a Eg&@sddlZddlZddlmZddlmZddlmZddlm Z ddl m Z ddl m Z ddlZddlZddlZddlmZd d ZGd d d eZdS) N)Config)get_pennseive_download_metrics)create_html_template) EmailSender)MonthlyStatsTable) itemgetter) relativedeltacCs<dd|D}t|}dd|D}t|tdddd}|S)NcSsg|]}t|qS)jsondumps.0dr r @/home/cmiss/Jenkins/workspace/SPARC-API/scripts/monthly_stats.py z%remove_duplicates..cSsg|]}t|qSr )r loadsr r r rrr datasetIdversionT)keyreverse)setsortedr)Zd_arrayZ json_listZjson_setZ unique_listr r rremove_duplicatess rc@seZdZd!ddZd"ddZd d Zd d Zd dZddZddZ ddZ ddZ ddZ ddZ ddZddZdd ZdS)# MonthlyStatsFcCst|_i|_tj|_d|_tj|_ d|_ ||_ ||_ |rF||_ ntj|_ tjdurzttj|_Wntyd|_Yn0dS)Nr)r send_grid user_statsrPENNSIEVE_ORGANIZATION organization_pennsieve_temp_api_keydatetimenow created_atZrun_day debug_email debug_modelogging_addressMETRICS_EMAIL_ADDRESS DATABASE_URLr monthlytableAttributeError)selfr&r%r r r__init__s    zMonthlyStats.__init__NTcCsJ|durtj}|jrF|j|rF|}|j||j||SgSN)r"r#dater*ZsendingRequiredrun pushStater)r,ZtimeNowcommitsendgrid_responsesr r rmonthly_stats_required_check+sz)MonthlyStats.monthly_stats_required_checkc sg}zj}WnDty`}z,td|d|WYd}~n0d}~00dfddjDd||S)Nz'Hit error while running monthly stats. z*Monthly stats sent successfully Sent to: cs,g|]$}dj|vrj|dqS)email)rkeys)r orcid_idr,r rrArz$MonthlyStats.run..z Send grid Responses: ) get_stats send_statsr BaseExceptionlogging exceptionsend_logging_email)r,r3errorr r8rr07s&zMonthlyStats.runcCsB||_|}||}||||_||_||jSr.) pennsieve_loginr!get_download_metrics_one_month"get_dataset_details_from_pennsievecreate_user_download_objectr&get_emails_orcid_id_map_from_pennsievepennsieve_user_details!user_stats_object_post_processing)r,metricsZ'dataset_details_for_downloaded_datasetsr r rr9Es   zMonthlyStats.get_statscCsxg}d}d}|D]P}d||vr||d}tt||d}|js|||}||q|jrt||g}|S)Nrr5datasets)r6rrr& send_emailappendr>)r,r responses email_address email_bodyr7rr r rr:Ns    zMonthlyStats.send_statscCsttddS)Nr)months)rrr8r r rrA^sz+MonthlyStats.get_download_metrics_one_monthcCsrttjd}||dd}|d}tjd|ddd}|jdtj tj d |d }|d d }|S) Nz/authentication/cognito-configZ tokenPoolZ appClientIdregionz cognito-idpr) region_nameaws_access_key_idaws_secret_access_keyZUSER_PASSWORD_AUTH)USERNAMEZPASSWORD)ZAuthFlowZAuthParametersZClientIdZAuthenticationResultZ AccessToken) requestsgetrPENNSIEVE_API_HOSTraise_for_statusr boto3clientZ initiate_authPENNSIEVE_API_TOKENPENNSIEVE_API_SECRET)r,rNZcognito_app_client_idZcognito_regionZcognito_idp_clientZlogin_responseapi_keyr r rr@bs"   zMonthlyStats.pennsieve_logincCs:tjtjd|jddd|jid}||S)Nz/organizations/z/members AuthorizationzBearer )headers)rUrVrrWr r!rXr )r,rNr r rrDzs z3MonthlyStats.get_emails_orcid_id_map_from_pennsievecCs`|jD]T}d|vr|dd}||jvr|d|j|d<|j|d|j|d<qdS)Norcidr5rH)rEr6r)r,userr7r r rrFs    z.MonthlyStats.user_stats_object_post_processingcCs@tdd|D}ttjdd|d}||dS)NcSsg|] }|dqS)rr r r r rrrzCMonthlyStats.get_dataset_details_from_pennsieve..z/discover/datasetsi)limitidsrH)rrUrVrrWrXr )r,rGZ uniqueIdsrNr r rrBsz/MonthlyStats.get_dataset_details_from_pennsievecsi}|D]|fdd|D}||}dD]P}|d}|dur2||vrhi||<|||d<q2||d|||d<q2q|S)Ncs g|]}d|dkr|qS)idrr r datasetr rrrz.Z contributorsr`rH)!add_dataset_name_to_download_infor6rV)r,Zdataset_details_objectZdownload_statsusers downloadInfoZ contributorr7r rerrCs   z(MonthlyStats.create_user_download_objectcCs(tdt|D]}|d||d<q|S)Nrname)rangelen)r,rfriir r rrgsz.MonthlyStats.add_dataset_name_to_download_infocCsF|jr |j}|jtj|d|StjdkrB|}|jtj|d|SdS)N&SPARC monthly dataset download summaryTRUE)r&r%r%sendgrid_email_with_unsubscribe_grouprr(SEND_MONTHLY_STATS)r,rLrMZemail_destinationr r rrIs   zMonthlyStats.send_emailc Cszx|jtj|jd|}|jdkr8td|jdn>|jdkrNtdn(|jdkrdtdntd |jWn.t y}zt|WYd}~n d}~00|S) Nrnz#Logging email sent successfully to z (202)iz=Could not send sendgrid email because rate limit is hit (403)izGCould not send sendgrid email. Sendgrid keys are likely incorrect (401)zUnknown error. Status code: ) rrprr(r' status_coder<infor?r;)r,messageresponseerrr r rr>s        zMonthlyStats.send_logging_email)Fr)NT)__name__ __module__ __qualname__r-r4r0r9r:rAr@rDrFrBrCrgrIr>r r r rrs     r)r<rY app.configrapp.metrics.pennsieverZ'scripts.monthly_downloads_html_templaterscripts.email_senderrZscripts.monthly_dbroperatorrrUr"r dateutil.relativedeltarrobjectrr r r rs