U
    Jj/                     @   s  d dl m Z  d dlmZ d dlZd dlmZmZ d dlZd dl	Z	d dl
mZ d dlZd dlZd dlmZmZmZmZ e  dZedZeed	d
Zi Zd!ddZdd Zdd Zd"ddZdd Zd#ddZdd Zdd Ze dkre Z!e"de!  e!se"d qe #d  qdS )$    )time)groupN)create_enginetext)load_dotenv)analyze_social_comments_batchanalyze_poll_comments_batchanalyze_comments_batch_genericget_social_prompt_version   	MYSQL_URIT)pool_pre_pingd   c              	   C   s4   d}t  }tjt||d| id}W 5 Q R X |S )Na  
    SELECT scr.*
    FROM social_comments_raw scr
    JOIN article_user_comment auc ON scr.comment_id = auc.external_comment_id
    WHERE auc.llm_sentiment IS NULL
    AND scr.article_id IS NOT NULL
    ORDER BY scr.comment_created_time ASC
    LIMIT :limit
    limit)params)engineconnectpdread_sqlr   )r   queryconndf r   W/var/www/api.dyntland.com/public_html/python_social_comments/process_social_comments.pyfetch_unprocessed_comments   s    

"r   c              	   C   s   t  }|tdd| i }W 5 Q R X |s4d S |d }|dkrx|d rx|d rx|d rxd|d |d |d gdS |d	kr|d
 rt  }|tdd| i }W 5 Q R X dd |D }|rd	|d
 |dS d S )Nz
            SELECT has_debate_poll, debate_ques, debate_option_yes, debate_option_no, poll_ques
            FROM article WHERE id = :article_id
        
article_idr   debate         )typequestionoptionspoll   z~
                SELECT title FROM article_poll_option
                WHERE article_id = :article_id ORDER BY id
            c                 S   s   g | ]}|d  r|d  qS )r   r   .0rr   r   r   
<listcomp>L   s      z(fetch_debate_context.<locals>.<listcomp>)r   r   executer   fetchonefetchall)r   r   Zarticlekindrowsr"   r   r   r   fetch_debate_context3   s0    

 

r.   c                 C   s*   t dddg}tt ddd}||fS )NABNeutralg333333?gffffff?r   )randomchoicerounduniform)commentr!   option_aoption_bside
confidencer   r   r   classify_comment[   s    r;   c                 C   s   ddddddddd	d
dg}| p d  }d|kr4d}nbd|krJ|dkrJd}nLd|ksZd|kr`d
}n6d|kspd|krvd}n d|ksd|krd}n
t|}t dk rt|}|S )zk
    Mock emotion classifier for testing pipeline.
    Later this will be replaced by real LLM output.
    ZangerZjoyZsadnessZfearZsurpriseZdisgustneutralZfrustrationZhopeZsarcasmZ	confusion ?!r/   ZlolZhahaZloveZgreatZhateZterribleg?)lowerr2   r3   )comment_textr9   	sentimentZemotionsr   baser   r   r   mock_llm_emotiong   s6    

rD   c                 C   s   |  dpd}|  dpd}zt|}W n   d}Y nX t|d d}t|d d}d| d	|  d	|  }t|d
 d dS )Ncomment_like_countr   comment_reply_count      ?
   r      g?g333333?r   r   )getfloatminr4   )rowr:   
like_countZreply_countZ
like_scoreZreply_scorerawr   r   r   compute_impact   s    
rP   rG   c                 C   s    z
t | W S    | Y S X d S )N)rK   )valdefaultr   r   r   
safe_float   s    
rS   c                   C   s  d} d}t dd}|jr$td d S g }|jdgd}|d}|D ]\}}|tkrbt|t|< t| }|rx|dnd }|dk}	|d	k}
|	r|d
 }|d d }|d d }n"|
r|d
 }|d }nd }d }d }|jdd}t	dt
|tD ]}|j||t  }|d  }|	r.t||||\}}n$|
rFt|||\}}nt|\}}|sztd| dt
|  |d7 }q|D ]t}z2|d}|d krW q~zt|ddd }W n" ttfk
r   Y W q~Y nX |dk s|t
|krW q~|j| }|	p|
}|r$t|dnd }|r8|dnd }|dd}|dd}t|dd}|dd}t||d k	r|nd}||d |||||||t||t d  W n8 tk
r } ztd!| W Y q~W 5 d }~X Y nX q~| t
|7 } qqDt| td" td# td$ td%t
| td&|d   td'|   td(|  td)t
|  t
d*d+ |D S ),Nr   r   )r   zNo new comments to processr   )subsetr    r   r#   r!   r"   r   T)dropcomment_messagez[LLM FAIL] article_id=z batch_size=indexr:   r9   rB   r1   emotionr<   toxicityg        language_codeenrG   
comment_id)r]   Zllm_assigned_sideZllm_confidenceZllm_sentimentZllm_emotionZllm_impact_scoreZllm_toxicity_scoreZllm_language_codeZllm_raw_outputZmodel_versionZllm_prompt_versionzMapping error:z
====================zPROCESSING SUMMARYz====================zFetched:zUnique articles:zTotal LLM outputs processed: zFailed batches: zTotal DB rows inserted: c                 S   s   h | ]}|d  qS )r]   r   r%   r   r   r   	<setcomp>F  s     z#process_comments.<locals>.<setcomp>)r   emptyprintdropnagroupbydebate_context_cacher.   rJ   reset_indexrangelen
BATCH_SIZEiloctolistr   r   r	   int
ValueError	TypeErrorrS   rP   appendjsondumpsr
   	Exceptioninsert_processednunique) total_processedZfailed_batchesr   resultsgroupedr   r   contextZctx_typeZ
has_debateZhas_pollr!   r7   r8   Zpoll_optionsiZbatch_dfcommentsZllm_outputsZ
used_modelitemidxrM   Zhas_sider:   r9   rB   rY   rZ   r[   Zimpacter   r   r   process_comments   s    


   
  






 r|   c              	   C   s8   | sd S t d}t }|||  W 5 Q R X t| S )Na[  
        UPDATE article_user_comment SET
            llm_assigned_side  = :llm_assigned_side,
            llm_confidence     = :llm_confidence,
            llm_sentiment      = :llm_sentiment,
            llm_emotion        = :llm_emotion,
            llm_impact_score   = :llm_impact_score,
            llm_toxicity_score = :llm_toxicity_score,
            llm_language_code  = :llm_language_code,
            llm_prompt_version = :llm_prompt_version,
            llm_raw_output     = :llm_raw_output,
            model_version      = :model_version
        WHERE external_comment_id = :comment_id
    )r   r   beginr)   rf   )r-   r   r   r   r   r   rq   u  s    
rq   __main__z[DEBUG] processed batch size: u   
✅ ALL COMMENTS PROCESSEDr   )r   )N)rG   )$r   tokenizer   pandasr   
sqlalchemyr   r   osr2   dotenvr   sysrn   Z
llm_clientr   r   r	   r
   rg   getenvDB_URIr   rc   r   r.   r;   rD   rP   rS   r|   rq   __name__	processedr`   sleepr   r   r   r   <module>   s<   

(
.
 E
