U
    Jj:                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m	Z	m
Z
 d dlmZ d dlmZmZmZ d dlZd dlZd dlZd dlmZ d dlZd dlZe  e dZe d	Ze d
ZdZdZdZe  Z!edddddddgdgdZ"ee"dZ#e!$de# e!$de# dd Z%dd Z&dd Z'd d! Z(d;d"d#Z)d$d% Z*d&d' Z+d(d) Z,d*d+ Z-d,d- Z.d<d.d/Z/d=d0d1Z0d2d3 Z1d4d5 Z2d6d7 Z3d8d9 Z4e5d:kre4  dS )>    N)HTTPAdapter)Retry)datetime	timedelta)load_dotenv)	urlencodeurlparseparse_qs)insert_dataframe_ignorePAGE_IDPAGE_ACCESS_TOKENZINSTAGRAM_BUSINESS_IDz https://graph.facebook.com/v19.0facebook	instagram      i  i  i  i  i  GET)totalZbackoff_factorZstatus_forcelistZallowed_methods)Zmax_retrieszhttps://zhttp://c                  C   sD   g } t s| d ts | d ts.| d | r@td|  d S )Nr   r   IG_BUSINESS_IDzMissing env vars: )r   appendr   r   
ValueError)missing r   Q/var/www/api.dyntland.com/public_html/python_social_comments/download_comments.py	check_env6   s    


r   c                 C   s   t | ddS )N%m/%d/%Yz%Y-%m-%d)r   strptimestrftime)Zdate_strr   r   r   
parse_dateA   s    r   c                 C   s,   | sdS t dt| } | jddddS )N ZNFKDutf-8replaceerrors)unicodedata	normalizestrencodedecode)textr   r   r   
clean_textD   s    r)   c                 C   s(   | sdS t |  }t|dk r$dS dS )NF   T)r%   striplen)r(   cleanedr   r   r   is_valid_commentJ   s    r.   c              
   C   s  |d kri }t |d< t d|  dt| }z*tj|dd}td |  | W S  t	j
jk
r   td td|   td	|j  td
|j d i  Y S  tk
 r } z6td td|   tdt| d i  W Y S d }~X Y nX d S )NZaccess_token/?   )timeoutg?z
[HTTP ERROR]z
Endpoint: zStatus: z
Response: 
z
[REQUEST FAILED]Error: )r   GRAPH_API_BASEr   sessiongettimesleepZraise_for_statusjsonrequests
exceptions	HTTPErrorprintZstatus_coder(   	Exceptionr%   )endpointparamsurlresponseer   r   r   fb_get]   s(    

rE   c                 C   s*   | sd S t d| }|r&t|dS d S )Nz*(?:dynt\.news|dyntland\.com)/article/(\d+)r   )researchintgroup)r(   matchr   r   r   extract_article_id   s    rK   c                 C   s   g }ddi}t | |d< t ||d< tt d|}||dg  |di }d|kr\qtt|d jd	 d
 }||d	< q$|S )NfieldszEid,created_time,permalink_url,message,comments.limit(0).summary(true)sinceuntilz/feeddatapagingnextafterr   )r   rE   r   extendr7   r	   r   query)rM   rN   ZreelsrA   rO   rP   rR   r   r   r   fetch_page_reels   s     
rU   c                 C   s   g }ddi}t | |d< t ||d< tt d|}||dg  |di }d|kr\qtt|d jd	 d
 }||d	< q$|S )NrL   z8id,timestamp,permalink,media_type,caption,comments_countrM   rN   z/mediarO   rP   rQ   rR   r   )r   rE   r   rS   r7   r	   r   rT   )rM   rN   mediarA   rO   rP   rR   r   r   r   fetch_ig_media   s     
rW   c              
   C   s   ddi}z*t |  d|}dd |dg D }W nH tk
rz } z*td|   tdt| d	 i }W 5 d }~X Y nX d
|dd
d
d
d
dS )NmetricZpost_impressions	/insightsc                 S   s"   i | ]}|d  |d d d qS namevaluesr   valuer   .0ir   r   r   
<dictcomp>   s      z*fetch_fb_reel_insights.<locals>.<dictcomp>rO   z
[INSIGHTS ERROR] reel_id=r4   r3   r   Z
reel_viewsZ
reel_reachZreel_sharesZreel_like_countZ
reel_saves)rE   r7   r?   r>   r%   )reel_idrA   rO   insightsrD   r   r   r   fetch_fb_reel_insights   s      
re   c                 C   st   ddi}z*t |  d|}dd |dg D }W n tk
rJ   i }Y nX d|dd|d	dd|d
ddS )NrX   zimpressions,reach,saved,sharesrY   c                 S   s"   i | ]}|d  |d d d qS rZ   r   r^   r   r   r   ra      s      z*fetch_ig_reel_insights.<locals>.<dictcomp>rO   r   ZreachZsharesZsavedrb   )rE   r7   r?   )media_idrA   rO   rd   r   r   r   fetch_ig_reel_insights   s    



rg   c                 C   s  g }ddd}d }|r||d< t |  d|}|dg }|sBq|D ]}||d f |dddkrt |d	  d|d
 dd}|dg D ]}	||	|d	 f q|rFt||krF|d |   S qF|di }
d|
krqtt|
d jdd gd }|sqq|d | S )Nd   z5id,message,created_time,from,like_count,comment_countlimitrL   rR   	/commentsrO   comment_countr   idrL   2   )rL   rj   rP   rQ   rE   r7   r   r,   r	   r   rT   )rc   rj   all_commentsrA   rR   rO   commentscommentrepliesreplyrP   r   r   r   fetch_fb_comments_with_replies   sH     ru   c                 C   s   g }ddd}d }|r||d< t |  d|}|dg }|s@q|D ]b}||d f |di dg }|D ]}	||	|d f qn|rDt||krD|d |   S qD|d	i }
d
|
krqtt|
d
 jdd gd }|sqq|d | S )Nrh   z-id,text,timestamp,username,like_count,repliesri   rR   rk   rO   rs   rm   rP   rQ   r   ro   )rf   rj   rp   rA   rR   rO   rq   rr   rs   rt   rP   r   r   r   fetch_ig_comments_with_replies  s2     rv   c                 C   s   |dkr| d n| d }t |d}|jdddd}| dpJ| dpJd	d d
 }t|d d d }|  d| S )Nr   created_time	timestampz%Y-%m-%dT%H:%M:%S%zr   )minutesecondmicrosecondmessagecaptionr   rn   r      _)	r   r   r    r7   hashlibmd5r&   	hexdigest	isoformat)postplatformtshourr(   Z	text_hashr   r   r   create_reel_fingerprintM  s     r   c                 C   sf  g }t d | D ] }t|d}dddddd}t|d }|D ]\}}t|ddp`|dd}	t|	snq@|d	i }
t|dd}||t	||t
|d t|ddd d
 |d |ddd||di di dd|d ||d k	|	|d |
dd|
dd|dd|ddd
 q@qt d |D ]}t|d}dddddd}t|d }|D ]\}}t|ddp|dd}	t|	sqpt|dd}||t	||t|d t|ddd d
 |d |ddd||dd|d ||d k	|	|dd|dd|dd|ddt|di dg d
 qpq@|S )Nu#   📝 Facebook comments + replies...r   r   rb   rm   r|   r   r(   from   rw   Zpermalink_url)
article_idZraw_jsonZreel_fingerprintr   rc   Z
reel_titlereel_created_timeZreel_urlrq   summarytotal_countr[   Z
like_countrl   )
Zreel_comments_count
comment_idZparent_comment_idZis_replycomment_messagecomment_created_timeZcomment_author_idcomment_author_nameZcomment_like_countZcomment_reply_countu$   📝 Instagram comments + replies...r   r}   rx   Z	permalinkcomments_countusernamers   rO   )r>   r   ru   r)   r7   r.   rK   r   r:   dumpsPLATFORM_FACEBOOKrv   PLATFORM_INSTAGRAMr,   )fb_reelsig_mediarowsreelfprd   rq   rr   Z	parent_idZcomment_textZauthorr   rV   r   r   r   process_all_comments[  s    


	







	




r   c                  C   sr   t jdd} | jdd dd | jdd dd |  }t }|tdd	 }|jpX|d
}|j	ph|d
}||fS )NzDownload FB/IG comments)descriptionz--sincez*Start date MM/DD/YYYY (default: yesterday))defaulthelpz--untilz$End date MM/DD/YYYY (default: today)r   )daysr   )
argparseArgumentParseradd_argument
parse_argsr   nowr   rM   r   rN   )parserargstodayZ	yesterdayrM   rN   r   r   r   get_date_range  s    r   c                  C   s   t   t \} }td|  d|  t| |}t| |}t||}dd }t|}||}|t	|d }tj
|d dd|d< t|d td	t| d
 |d jdd}td t| d S )NzFetching comments from z to c                 S   sJ   ddg}|D ]8}|| j krtj| | dd| |< | | jd| |< q| S )Nr   r   coercer!   z%Y-%m-%d %H:%M:%S)columnspdto_datetimedtr   )dfZdatetime_colscolr   r   r   format_datetime_columns  s    
z%main.<locals>.format_datetime_columnsr   r   r!   social_comments_rawu   ✅ Inserted z rows into DBr   F)dropnaz
Platform Breakdown:)r   r   r>   rU   rW   r   r   	DataFramewherenotnull
to_numericr
   r,   value_counts)Z	FROM_DATEZTO_DATEr   r   rO   r   r   Zplatform_countsr   r   r   main  s     





r   __main__)N)N)N)6osr;   csvr8   r   Zrequests.adaptersr   Zurllib3.util.retryr   r   r   dotenvr   urllib.parser   r   r	   pandasr   r#   rF   dbr
   r:   r   getenvr   r   r   r5   r   r   ZSessionr6   Zretry_strategyadapterZmountr   r   r)   r.   rE   rK   rU   rW   re   rg   ru   rv   r   r   r   r   __name__r   r   r   r   <module>   sd   




*
1
.a9
