o }h @sNddlZddlmZddlmZmZmZddlmZm Z ddl m Z m Z m Z mZmZddlmZmZmZmZmZmZddlmZmZddlmZmZdd lmZddlZddl Z ddl!Z!ddl"Z"ddl#Z#ddl$Z$ddl%Z%ddl&Z&dd l'm(Z(ddl)Z)ddl*Z*ddl+Z+ddl,Z,ddl-Z-dd l.m/Z/dd l0m1Z1dd l2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddlm?Z?m@Z@mAZAmBZBddlCmDZDddlEmFZFddlGmHZHddlImJZJddlKmLZMddlNmOZOddlPmQZQddlRmSZSddlTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdddlemfZfmgZgmhZhmiZimjZjmkZkddllmmZmddlnmoZodd lpmqZqdd!lrmsZsmtZtmuZumvZvmwZwdd"lxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZdd#lmZdd$lmZmZmZdd%lmZdd&lmZdd'lmyZmZddl-Z-e*e?eZeqjZejee*ee*je;d(d)Zeqjejd*<eSed+d,d-d.ZeDeeFeZefZdae!jd/eqjeqjd0d1ZemZeqjZered2red2d3d4ZzeseZWn eydZYnwzeteZWn eydZYnwzeueZWn eydZYnwzeveZWn ey+dZYnwzeweZWn ey>dZYnwGd5d6d6eZed7d8d9Zejd:d;Zejdkredused?d@kreejse*dAeejse*dBeere1Ze ejejdCdDdDdEdEdddFeqjdGkreqjd>krejse*dHeejedCdIdJdKiZejdLdMZiZeZe ZeZejdNdOZeZejdPdQZejedRd4dSejedRd4dSejedRdTdUe3e5e7d4dVgZedWdXeΡeqjd>kr=ejedCdYdddZd[d\Zeeϡed]d^d_Zejd`dagdbdcddZdedfZedgdheqjfdidjZedkdldmZedneqjfdodpZedqeqjfdrdsZdtduZedvdwdxZdydzZed{eqjfd|d}Zed~ddZededddZedddZedddZededddZededddZededddZedddZedddZedddZedddZddZddZddZedddZedeqjfddZedddZddZejddddddeddwddZejdddidedddZedddÄZddńZddDŽZddɄZedʡdd̄Zed͡ddτZedСdd҄ZedӡddՄZed֡dd؄Zejddgdbdd܄Zejddgdbdd߄Zejddgdbejd,dddZejddgdbddZddZejddgdbddZejddgdbddZejddgdbddZejddgdbddZejddgdbddZejddgdbdxddZejddgdbddZejddgdbddZejѐddgdbddZ ejѐddgdbddZ dd Z d d Z d d Z ejѐddagdbddZejѐddagdbddZejѐddagdbddZejѐddagdbddZejѐddagdbddZejѐddagdbddZd d!Zdyd"d#Zejѐd$dagdbd%d&Zd'd(Zdzd)d*Zd+d,Zd-d.Zejѐd/dagdbd0d1Zejѐd2dagdbd3d4Zejѐd5dgdbd6d7Zejѐd8dagdbd9d:Zd;d<Zd=d>Z ejѐd?dagdbd@dAZ!eѐdBdCdDZ"eѐdEdFdGZ#eѐdHdIdJZ$ejѐdKdagdbdLdMZ%ejѐdNdagdbdOdPZejѐdQdagdbdRdSZejѐdTdagdbdUdVZ&eѐdWdXdYZ'ejѐdZdgdbd[d\Z(ejѐd]dgdbd^d_Z)ejѐd`dgdbdadbZ*ejѐdcdgdbdddeZ+ejѐdfdagdbdgdhZ,ejdidejѐdjdgdbdkdlZ-ejdidejѐdmdgdbdndoZ.eѐdpejdqddrdsZ/ejѐdtdagdbdudvZ0dS({N)get_download_count)init_cf_cda_clientget_funded_projects_countget_featured_datasets)update_all_events_sort_orderupdate_event_sort_order)get_dataset_countinit_algolia_clientget_all_dataset_idsget_all_dataset_uuidsget_associated_datasets)init_ga_reportingget_ga_1year_sessionsinit_gspread_clientappend_contact upload_fileinit_drive_client)set_featured_dataset_id#get_featured_dataset_id_table_state)update_protocol_metrics get_protocol_metrics_table_state)OSparcServices) ApiException)urlparse)BackgroundScheduler) OrTrigger) DateTrigger)IntervalTrigger) ClientError)ThreadPoolExecutor)datetime timedelta)Flaskabortjsonifyrequest)CORS) Marshmallow) Pennsieve) new_client)UnauthorizedException)Image) HTTPBasicAuth)Cache)create_doi_querycreate_filter_requestcreate_facet_querycreate_doi_aggregatecreate_title_querycreate_identifier_query!create_pennsieve_identifier_querycreate_field_querycreate_request_body_for_curiescreate_onto_term_querycreate_multiple_doi_query create_multiple_discoverId_querycreate_anatomy_queryget_body_scaffold_dataset_idcreate_multiple_mimetype_querycreate_citations_query) EmailSenderfeedback_emailissue_reporting_email#creation_request_confirmation_email-anbc_form_creation_request_confirmation_email2service_form_submission_request_confirmation_email)Lock) ElementTree)Config)AnnotationTableMapTable ScaffoldTableFeaturedDatasetIdSelectorTableProtocolMetricsTable)process_resultsprocess_get_first_scaffold_inforeform_aggregation_resultsreform_curies_resultsreform_dataset_resultsreform_related_termsreform_anatomy_results)ContactRequestSchema)img_to_base64_strget_path_from_mangled_list get_extension)start_simulation)check_simulation)rLprocess_result)Z max_workersZENVZ SimpleCache,)Z CACHE_TYPEZCACHE_DEFAULT_TIMEOUT)configs3z us-east-1)Zaws_access_key_idZaws_secret_access_keyZ region_namez postgres://z postgresql://c@s`eZdZdZeedZdZe ddZ ddZ e dd Z d d Z e d d ZddZdS) Biolucidai?BFcC |t_dSN)r__tokenvaluerf3/home/cmiss/Jenkins/workspace/SPARC-API/app/main.py set_token zBiolucida.set_tokencC|jSrb)rcselfrfrfrgtokenzBiolucida.tokencCrarb)r_ _expiry_daterdrfrfrgset_expiry_daterizBiolucida.set_expiry_datecCrjrb)rorkrfrfrg expiry_daternzBiolucida.expiry_datecCrarb)r__pending_authenticationrdrfrfrgset_pending_authenticationriz$Biolucida.set_pending_authenticationcCrjrb)rrrkrfrfrgpending_authenticationrnz Biolucida.pending_authenticationN)__name__ __module__ __qualname__rcr nowr!rorr staticmethodrhrmrprqrsrtrfrfrfrgr_{s    r_cCstt|ddfS)N)errorrz)r$str)erfrfrgresource_not_foundsr~c CszttjtjdtjdaWdStjjy-}zt dt |WYd}~dSd}~wt yI}zt dt |WYd}~dSd}~wt ye}zt dt |WYd}~dSd}~ww)NF)Z api_token api_secretZ env_overridehostz#Unable to connect to Pennsieve hostz&Unable to authorise with Pennsieve Apiz Unknown Error) r(rFPENNSIEVE_API_TOKENPENNSIEVE_API_SECRETPENNSIEVE_API_HOSTpsrequests exceptions HTTPErrorloggingr{PSUnauthorizedException Exceptionerrrfrfrgconnect_to_pennsieves(    rc CsTzttjtjtjddaWdSty)}ztd|WYd}~dSd}~ww)N)api_keyrZapi_hostZ api2_hostz'Error connecting to pennsieve 2 agent: ) r)rFrrrps2rrr{rrfrfrgconnect_to_pennsieve2s rZFALSEtotal_protocol_viewsz3Starting scheduler for protocol metrics acquisitionz6Starting scheduler for featured dataset id acquisitionZcron*2)yearmonthdayhourminutesecondZ developmentz8Starting scheduler for updating contentful event entriesz US/Eastern)rtimezonec Cstdz&tjtjdd}|jr*d|jddvr*|}t |d}|t d<Wnt yC}z t d |WYd}~nd}~wwt jsRtd t dSdS) NzGetting oSPARC viewersz/viewersurlapplication/jsonz content-typer`data file_viewersz!Could not retreive oSPARC viewersz1Starting scheduler for oSPARC viewers acquisition)rinforgetrFOSPARC_API_HOSTokheadersjsonbuild_filetypes_table osparc_datarr{viewers_schedulerrunningstart)reqZviewerstabler}rfrfrgget_osparc_file_viewerss    rcCsttdtrtt}|td<trtt}|td<tr#tt}|td<t }|td<t j s8tdt dSdS)NzGathering metrics dataZ1year_sessions_countZ dataset_countZfunded_projects_countZ1year_download_countz*Starting scheduler for metrics acquisition) rrgoogle_analyticsr usage_metricsalgoliar contentfulrrmetrics_schedulerrr)Z ga_responseZalgolia_responseZ cf_responseZ ps_responserfrfrg get_metricss   rc Csntdztjtjdd}|}t|dWdSt y6}z t d|WYd}~dSd}~ww)NzFetching oSPARC servicesz /servicesrrz%Request to get oSPARC services failed) rrrrrFrrosparc_servicesZ set_servicesrr{)rZ services_respr}rfrfrg get_servicess rinterval)functriggerdays)rrhours)rcCttSrb)rfeaturedDatasetIdSelectorTablerfrfrfrg'srsat)Z day_of_weekrrcCstdtjr ttdtjrttdtjr$ttdtjr0ttdtjrz#Could not find thumbnail in file: 'r`rZRGBrowscols)r%rr#rr]rrrreaddecodelenfindrE fromstringattribtextbytesfromhexr+ frombytesintrT)rrrrresourceZstart_tag_foundZ end_tag_foundZ start_byteoffsetZend_byterexZstart_thumbnail_elementZ thumbnail_xmlxmlZ size_infoZim_datachildZ byte_im_dataZimZ base64_formrfrfrgextract_thumbnail_from_xml_files`         "rz/exists/cCsftj}|d|}z tj||dd}Wn ty ddiYSw|dd}|dkr/ddiSddiS) Nrrrrrexistsfalse ContentLengthrtrue)r%rrr] head_objectr)rrrr head_responsecontent_lengthrfrfrg url_existss    rcCs0|dd}|d|d}t|}|S)Nzhttps://api.pennsieve.io/z"https://api.pennsieve.io/discover//)replacerfindrrr)urirrfrfrgfetch_discover_file_informations  r%z/s3-resource/discover_pathc Cstjd}zt|}d|vr |ddkr |dd}|dWSWnty9}z td|WYd}~nd}~wwtdd |d S) Nr#Z totalCountr^filesrrzFailed to retrieve uri {uri}rzzFailed to retrieve uri r)r%rrr%rrr{r#)r# json_response file_inforrfrfrgget_discover_paths   r)c Csz&tj||dd}|dtj}|r!|tjks$tdd|dWSWdSWdStjjyk}z7|j ddd krAdWYd}~S|j ddd krRdWYd}~St|j dd|j ddWYd}~Sd}~ww)NrrrizFile too big to download: rrrZ404rz.Provided path was not found on the s3 resourceZ403EThere is a permission issue when accessing the file at specified pathMessage)rOK)rzr*)r+r,) r]rrrFZDIRECT_DOWNLOAD_LIMITr#botocorerrr)rrrrrrfrfrgs3_header_checks*  *r0z/s3-resource/c Cstj}|d|}|}t||}|ddks|ddkrQt|}||kr*tdddt||}|ddkr8|}n|ddkrEtdddn |ddkrQtdddtj||d d }tjd }|d } |durnt | S| S) Nrrrzr+r*rrr,rrZ encodeBase64r) r%rrr0rUr#r]rrrr) rrrrZs3_pathrZs3_path_modifiedZ response2 encode_base64rrfrfrgdirect_download_url s2           r2z /scicrunch-dataset//c Cs|ddd|}t|}ztjtjdtj|d}|WStjj yA}zt |t d|iWYd}~Sd}~ww)NDOI:r`r /_search?api_key=rr{) r!r.rpostrFSCI_CRUNCH_HOSTKNOWLEDGEBASE_KEYrrrrr{r)Zdoi1Zdoi2doirrrrfrfrgsci_doi0s  r:z /pubmed/z/pubmed//c Cs\z td|d}|jWStjjy-}zt|td|iWYd}~Sd}~ww)Nz https://pubmed.ncbi.nlm.nih.gov/r r{) rrr rrrr{rr)id_rrrfrfrgpubmed@s r<z/scicrunch-query-string/c Cstjd}tjd}tjd}tjd}t||||}ztjtjdtj |d}t | WStj j yT}zt|t d|iWYd}~Sd}~ww)Nfieldcuriesizefromr4r5r{)r%rgetlistrr5rr6rFr7r8rLrrrrr{r)fieldsr>r?Zfrom_rrrrfrfrg sci_organLs     rCz/dataset_info/using_doicCs<tjd}tjd}t|}|durtt|St|S)Nr9Z raw_response)r%rrr.rPdataset_search)r9rawqueryrfrfrgget_dataset_info_doi_s   rGz!/dataset_info/using_multiple_doisz"/dataset_info/using_multiple_dois/cC tjd}t|}tt|S)Ndois)r%rrAr8rLrD)rIrFrfrfrgget_dataset_info_doisk  rJz./multiple_dataset_info/using_multiple_mimetypez//multiple_dataset_info/using_multiple_mimetype/cCrH)Nq)r%rrAr<rLrD)rLrFrfrfrgget_file_info_from_mimetypets  rMz(/dataset_info/using_multiple_discoverIdsz)/dataset_info/using_multiple_discoverIds/cCrH)N discoverIds)r%rrAr9rLrD)rNrFrfrfrgget_dataset_info_discoverIds~rKrOz/dataset_info/using_titlecCrH)Ntitle)r%rrr2rPrD)rPrFrfrfrgget_dataset_info_title  rQz%/dataset_info/using_object_identifiercCrHN identifier)r%rrr3rPrDrTrFrfrfrg"get_dataset_info_object_identifierrRrVz/dataset_info/anatomycCs6tjdd}|dkrtdddSt|}tt|S)NrTrrzz Identifier for API call not set.r)r%rrr#r:rRrDrUrfrfrgget_dataset_info_anatomys   rWz(/dataset_info/using_pennsieve_identifiercCrHrS)r%rrr4rPrDrUrfrfrg%get_dataset_info_pennsieve_identifierrRrXc Csg}|D]}|r|d|ks|r|didd|kr|di}|did} |di} | d} | d} |d id} | rX|||d |dd|| d | dur| D]7}|d d }t|||d||}t|dkr{||q^| dus| dur||d |dd|d}||q^| dur| durtt| D]%}d|krt| krnq| |dd}|td|d| ||qq|S)NrTdatasetrr`dataciteZ isDerivedFromZderived_from_datasetr#Zassociated_flatmapr) discoverIdrrr flatmapUUIDz'derivative/sub-f006/derivative/sub-f006zderivative/sub-f006r)r[rrrzhttps://doi.org/)rappendr!+get_is_derived_from_with_identifier_or_pathrextendrangeget_original_source)r[robjectsrT matchingPath datasetCacheZ source_listitemrZZisDerivedFromPathsZderivedfromdatasetZderivedfromdatasetdoiZderivedfromdatasetpathr\rZsource_objectsrir9rfrfrgr^sT           r^cCs|didg}t|dkrR|ddid}|ddidid}|ddididid}|durR|durR|durRt||||||SgS) Nhitsr^r_sourcerb pennsieverTr)rrr^) dataset_inforTrcrdrgrbr[rrfrfrgget_original_source_in_datasets &rkc Csd}|}|dur t|}n!|dur.|dur#|}|dd}t|g}n |dur.|}t|g}||d}|dur@t|}|||<t||||S)Nr3r`)r3r!r8r9rrDrk) rTr9r[rrdrFidZnewDOIrjrfrfrgras"     raz/file_info/get_original_sourcecCsHtjd}tjd}tjd}tjd}i}dt|||||iS)Nr[r9rTrresult)r%rrra)r[r9rTrrdrfrfrgget_file_info_original_sources    rnz/segmentation_info/c CsLtj}d|vr tdddS|d|}|d}zt||}|ddkr*tdddtj||d d }Wn5tyh}z)|jd d d krTtdd|ddWYd}~Stdd|ddWYd}~Sd}~ww|d }t |}| d}i} |dur|j | d<n ddddd| d<| d} | dur| j | d<| Sddi| d<| S)N dataset_pathrz,Query arguments must include 'dataset_path'.rrrrzr*rrrrrrrrrz./{*}sparcdata/{*}subjectsubjectr`)ZageZsexspeciesZ subjectidz./{*}sparcdata/{*}atlasZatlasZorgan)r%rr#rr0r]rrrrrEr r r ) rrrrorrrrZsubject_elementrZ atlas_elementrfrfrgget_segmentation_info_from_filesB                rrz/current_doi_listcCsBt}tt|}g}|ddD] }||ddqd|iS)Nr9bucketsrr>results)r1rNrDr])rFrtZ doi_resultsrmrfrfrg get_all_doi0s  ruc Cspz|}dtji}tjtjd||d}|WStjjy7}zt |t d|iWYd}~Sd}~ww)Nr/_search)rrr{) rFr8rr6r7rrrrr{r$)rFpayloadrrrrfrfrgrD;s  rDz/search/r` )rFlimitr)defaultsz/search/c CszFtjddurtjd}tjddurtjd}tjddur+tjd}ttjd|d|d|dtj }t|WStj j yf}zt |t d|iWYd}~Sd}~ww) NryrFrz /_search?q=z&size=z&from=z &api_key=r{)r%rrrrFr7r8rLrrrrr{r)rFryrrrrfrfrg kb_searchMs   * r{z/filter-search/rFz/filter-search//c Cstjd}tjd}tjd}tjd}t|||||}ztjtjdtj |d}t | }W|Stj j y[}zt|tt|ddd fWYd}~Sd}~wt jymtd d d d fYSw) NtermZfacetr?rr4r5zc Cst|\}}g}||D]U}||d|dd<ztjtjdtj|d}|}||Wq tjyGt dddd fYSt ya}zt d ||WYd}~q d}~wwg}|D] } || d|d 7}qft |S) N aggregationsrr=r4r5r~rrrz$Could not search SciCrunch for path rs) r0rr6rFr7r8rr]rr$rrr{) type_Ztype_maprrtrrZ json_resultrrrmrfrfrg get_facets|s2   rcCs*d|vrt|d}|j|d<dSdS)NZreadmemarkdown)rrr )respZmark_reqrfrfrginject_markdownsrc Cs@|d}|d}|dus|durdSt|}|j}z tj|d|dd}Wn>tyftjdk}|r;t dz tj|d|dd}Wntyc}z|rWt |WYd}~YdSd}~wwYnw|d  }zt |} Wnty}z t |WYd}~dSd}~ww| d | d | d d |d<dS)Nrlr#z{}/files/template.jsonrrZTRUEzXRequired file template.json was not found under /files folder, trying under /packages...z{}/packages/template.jsonruuidrr)rrrZstudy)rrnetlocr]rformatrrFSPARC_API_DEBUGGINGrwarningr{rrr ValueError) rr;r#Z parsed_uribucketr debuggingr}templateZ template_jsonrfrfrginject_template_datasX        rcCsFi}|D]}|d}|d=||dsg||<|||q|S)N file_typeF)lowerrr])Zosparc_viewersrZviewerZfiletyperfrfrgrs  rz/sim/dataset/c CstjdkrGz tdtj|}|jr$|}t |t |t |WSWnt y@}zt d||WYd}~nd}~wwtdddSdS)NGETz{}/datasets/{}Could not fetch SIM dataset rzResource not foundrr%methodrrrrFDISCOVER_API_HOSTrrrrr$rrr{r#)r;r json_datarrfrfrg sim_datasets   rz&/sim/dataset//versions/c CstjdkrKz!tdtj||}|jr%|}t |t |t |WSWnt yD}zt d|d||WYd}~nd}~wwtdddSdS)Nrz{}/datasets/{}/versions/{}rz version rzrrr)r;Zversion_rrrrfrfrgsim_dataset_versionss  $ rz/get_osparc_datacCrrb)r$rrfrfrfrgget_osparc_datarz /sim/servicecCsTtjdkr(tjd}tjjddtd}tjjddtd}t|||}t|SdS)Nrsearchryr)defaulttypeskipr)r%rrrrrZsearch_servicesr$)rryrrtrfrfrg osparc_searchs  rz /sim/filecCs tjdkrtj}td|iSdS)Nrr)r%rrZfile_extensionsr$) extensionsrfrfrgosparc_extensionss rz/project/rcCs4t|}t|ddkrt|dStddddS)Nrgrrzrr)r rr$r#)Z project_iddatasetsrfrfrgdatasets_by_project_ids rz"/get_featured_datasets_identifierscCs dtiS)N identifiers)rrfrfrfrg!get_featured_datasets_identifiersrirz/get_featured_dataset)rc Csttd}|dkr d}z!tdtj|}|dgkr+tdtjd}|WStyH}zt d||WYd}~nd}~wwt ddd dS) Nfeatured_dataset_idr z{}/datasets?ids={}rzCould not get featured dataset rzz,An error occured while fetching the resourcer) rrrrrrFrrrrr{r#)rrrrfrfrgget_featured_datasets  rz/reva/subject-idsc Csz/tdtj}|d}g}|D]}|dddkr%||ddqtd|dd fWStyR}ztd |td |d d fWYd}~Sd}~ww) N /packages/childrenr packageType Collectionrsuccess)ridsrz+Error while getting REVA subject id files: rr) rrrF,REVA_3D_TRACING_PRIMARY_FOLDER_COLLECTION_IDr]r$rrr{)primary_folderprimary_childrenZ subject_idsrr}rfrfrggetRevaSubjectIds1src sXzddtdtj}|s#dtj}t|td|dWS|dg}tfdd |Dd}|durJd }t|td|dWStd|d d }|smd |d d }t|td|dWS|dg}tfdd |Dd}|durd}t|td|dWStd|d d }|sd|d d }t|td|dWS|dg} tfdd | Dd} | durd}t|td|dWStd| d d } | sd| d d }t|td|dWS| dgWSty+} zd| }t|td|dWYd} ~ Sd} ~ ww)NZCoordinatesDataZInSiturzPrimary folder not found: rzrrc3$|] }|ddkr|VqdSrrNrf.0r subject_idrfrg K"z5getRevaTracingInSituFolderChildren..z)Subject folder not found for subject id: rrlz,Subject folder could not be fetched for id: c3rrrfr)coordinates_folder_namerfrgrXrz.CoordinatesData folder not found for subject: z4CoordinatesData folder could not be fetched for id: c3rrrfr)in_situ_folder_namerfrgrerz%InSitu folder not found for subject: z+InSitu folder could not be fetched for id: z2Exception thrown when getting Reva InSitu Folder: r) rrrFrrr{r#nextr) rrmsgr subject_childsubject_foldersubject_childrenZcoordinates_childZcoordinates_folderZcoordinates_childrenZ in_situ_childZin_situ_folderr}rf)rrrrg"getRevaTracingInSituFolderChildren@s`                rz-/reva/anatomical-landmarks-files/c szdt|}tfdd|Dd}|dur1tdd|tdd|dd fWStd |d d }|d }g}|D]_}|d d}|d d }td |} | d } g} | D]6} | d d } td | d}|dd d }td | d|d}| t| d dt|dqc|t|| dqEtd|ddfWSt y}ztd|td|ddfWYd}~Sd}~ww)NZAnatomicalLandmarksc3rrrfrZ anatomical_landmarks_folder_namerfrgrrz2getRevaAnatomicalLandmarksFiles..REVA tracing folder  not found for subject: ERROR folder not found for subject: rrzrrrlrr/viewr/files/r)rs3Url)rr&r)rfoldersrz4Error while getting REVA anatomical landmarks files z0Error while getting anatomical landmarks files: r rrrr{r$rrr]r|r)rin_situ_childrenZanatomical_landmarks_childZanatomical_landmarks_folderZanatomical_landmarks_childrenZanatomical_landmarks_foldersZanatomical_landmark_childZlandmark_folder_nameZlandmark_folder_idZanatomical_landmark_folderZlandmark_childrenZlandmark_filesZlandmark_childZlandmark_file_package_idZ landmark_fileZlandmark_file_idZlandmark_file_presigned_urlr}rfrrggetRevaAnatomicalLandmarksFileszs:   "rz /reva/tracing-files/c szdt|}tfdd|Dd}|dur1tdd|tdd|dd fWStd |d d }|d }g}|D]R}td |d d }|d }|D]=} | d d } td | d} | dd d } td | d| d} |t| d dt|d dt| dqYqEtd|ddfWSt y}ztd|td|ddfWYd}~Sd}~ww)NZ VagusNervec3rrrfrZvagus_nerve_folder_namerfrgrrz&getRevaTracingFiles..rrrrrrzrrrlrrrrrr)rregionrrrr&rz'Error while getting REVA tracing files z#Error while getting tracing files: rr)rrZvagus_nerve_childZvagus_nerve_folderZvagus_nerve_childrenZvagus_tracing_filesZvagus_region_childZvagus_region_folderZvagus_region_childrenZvagus_file_childfile_package_idZ vagus_fileZ vagus_file_idZvagus_file_presigned_urlr}rfrrggetRevaTracingFiless8 &rz!/reva/micro-ct-files/c sdztdtj}|d}tfdd|Dd}|dur8tdtddd d fWStd|d d }|d}tfd d|Dd}|durrtddtddd d fWStd|d d }|d}g} |D]S} | d d } td| d} | dd d } td| d| d}| d d}| d}| d d}| dd d}| t |t |t |t |t |dqtd| ddfWSt y}ztd|td|d dfWYd}~Sd}~ww)Nz-MicroCTVisualizationrrc3rrrfrrrfrgrrz&getRevaMicroCtFiles..z/REVA microCT folder not found with subject id: rz*MicroCT folder not found with subject id: rrzrrlc3rrrfr)"micro_ct_visualization_folder_namerfrgrrz REVA microCT rrrrrrZstoragerZfileType)rrrrr?rrrz'Error while getting REVA microCT files z#Error while getting microCT files: r) rrrFZ*REVA_MICRO_CT_PRIMARY_FOLDER_COLLECTION_IDrrr{r$r]r|r)rrrrrrZmicro_ct_childZmicro_ct_visualization_folderZmicro_ct_childrenZmicro_ct_filesZ micro_childrZmicro_child_fileZmicro_child_file_idZmicro_file_presigned_url file_name file_sizeZ package_typerr}rf)rrrggetRevaMicroCtFilessF    "rz/get_owner_email/csNtjj}tjj|}fdd|D}|stddddStd|djiS)Ncsg|] }|jkr|qSrf)Zint_id)rxowner_idrfrg sz#get_owner_email..rzzOwner not foundrrr)rZ_apiZ _organizationZ organizationsZ get_membersr#r$r)rorgmembersresrfrrgget_owner_emails rz!/get_body_scaffold_info/cCs:t|}|rt|}tt|}|r|Stdd|dS)NrzzWhole body info not found for r)r;r4rMrDr#)rqrlrFrmrfrfrgget_body_scaffold_infos rz/thumbnail/Fc Cst}zZt|stWdn1swYtjd|}d|i}tjd||d}t |j }|dkr[|s[t | dWdn1sQwYt |d}|WStyx}ztd||WYd}~nd}~wwtd d S) Nz/thumbnail/{0}rmrrseyJzdGF0dXMiOiJBZG1pbiB1c2VyIGF1dGhlbnRpY2F0aW9uIHJlcXVpcmVkIHRvIHZpZXcvZWRpdCB1c2VyIGluZm8uIFlvdSBtYXkgbmVlZCB0byBsb2cgb3V0IGFuZCBsb2cgYmFjayBpbiB0byByZXZlcmlmeSB5b3VyIGNyZWRlbnRpYWxzLiJ9r`Tz Could not get the thumbnail for rzz-An error occured while fetching the thumbnail)r_biolucida_lockrmauthenticate_biolucidarFBIOLUCIDA_ENDPOINTrrr%rrrrhthumbnail_by_image_idrrr{r#)image_idZrecursive_callblrrrencoded_contentrrfrfrgrs2    rz/image/c Csltjd|}z td|}t|WSty0}zt d||WYd}~nd}~wwt ddS)Nz/image/info/{0}rzCould not get image info for rzz/An error occured while getting the image's info) rFrrrr%process_biolucida_resultrrrr{r#)rrrrrfrfrgimage_info_by_image_ids  rz/image_search/c Csjtjd|}z td|}|WSty.}ztd||WYd}~nd}~wwddidfS)Nz%/imagemap/search_dataset/discover/{0}rz$Could not search images for dataset r{z3An error occured while searching images for datasetrz) rFrrrr%rrrr{) dataset_idrrrrfrfrgimage_search_by_dataset_id"s   rz/image_xmp_info/cCsvtjd|}ztd|}Wntjjy!tdddYSw|}|ddkr2t |dStdd |dS) Nz/image/xmpmetadata/{0}rr)Unable to make a connection to Biolucida.rrrrzXMP info not found for ) rFrrrr%rrr#rprocess_biolucida_resultsrrrmrrfrfrgimage_xmp_info-s  rz/image_blv_link/cCsztjd|}ztd|}Wntjjy!tdddYSw|}|ddkr4t d|diStdd |dS) Nz/image/blv_link/{0}rrrrrrlinkzBLV link not found for ) rFrrrr%rrr#rr$rrfrfrgimage_blv_link<s rcCsht}tjd}tjtjdd}g}i}tjd||||d}|jtjj kr2| }| |ddSdS)Nz /authenticater`)usernamepasswordrmr)rrr&rm) r_rFrZBIOLUCIDA_USERNAMEZBIOLUCIDA_PASSWORDrr%rcodesrrrh)rrrwr&rrrrfrfrgrKs rcCs\d}|r&t}|rd|vr|d}|||}td|iStddddStddddS) NTstaterrzState not specifiedrrzDatabase not available)r%get_jsonZ pushStater$r#)rZcommitrrrrfrfrgget_share_link\s   rcCs`|r(t}|r d|vr |d}||}|r td||iStddddStddddS)Nrrrz#Key missing or did not find a matchrrzr)r%rZ pullStater$r#)rrrrrfrfrgget_saved_statejs  rz/annotation/getshareidcCrrb)rannotationtablerfrfrfrgget_annotation_share_linkxrrz/annotation/getstatecCrrb)rrrfrfrfrgget_annotation_state~rrz/map/getshareidcCrrb)rmaptablerfrfrfrgget_map_share_linkrrz /map/getstatecCrrb)rrrfrfrfrg get_map_staterrz/scaffold/getshareidcCrrb)r scaffoldtablerfrfrfrgget_scaffold_share_linkrr z/scaffold/getstatecCrrb)rrrfrfrfrgget_scaffold_staterr c Csz&tjtjtj|dd}|}d|vs|ds ddidfWS|ddWSty?}z t d|WYd}~dSd}~ww) NZsecretrrrrr{Failed Captcha ValidationF0Could not validate captcha, bypassing validation) rr6rF TURNSTILE_URLNUXT_TURNSTILE_SECRET_KEYrrrrr{)rm captchaReq captchaResprrfrfrgverify_recaptchasrc Csdtjdtjd}dtjdd}||d}|r||d<|r%||d <tj|||d }|jd krB|}|d |d |ddStd|j )Nhttps://api.github.com/repos/r z/issuestoken application/vnd.github+json AuthorizationAccept)rPbodylabels assigneesrrhtml_url comments_urlr)r r! issue_api_urlzGitHub Issue creation failed: ) rFSPARC_GITHUB_ORGSPARC_ISSUES_GITHUB_REPOSPARC_TECH_LEADS_GITHUB_TOKENrr6rrrr ) rPrrrrrrrZ response_jsonrfrfrgcreate_github_issues&  r&z /create_issuec CsPtj}tjd}tjds|rt|stddidfS|dd}|d}|d }|r0|s6tdd d |d d }|dvryzt |||gt j d}|d}|d}|d} Wn't yx} ztdt | idfWYd} ~ Sd} ~ wwtdd|idfSd} d} d} |rd} d} d} tj}|rSd|vrS|d}|}|j}td}tj}|d|d|}dt jdt jd|}dt jd d!d"}t|d#}d$||d%}zQtj|||d&}|jd'vr.| }|d(d)}d*|d+}dt jd d,}d |i}tj!|||d-}|jdkr)| d.7} d} d/} n | d07} n| d17} d} d/} Wnt yR} z| d17} d} d/} WYd} ~ nd} ~ ww|rd2}t"#d3|i}|d4krmd5}t$#d3|i}t%%|}zt&'t j(|||| d67} Wnt y} z| d77} d} d/} WYd} ~ nd} ~ wwt| || | d8| fS)9N captcha_tokenTESTINGr{Invalid reCAPTCHArrbugrPrzMissing title or bodyrrr`)r*feedbacktest)rrr r!r"rUnsupported task type: zSubmission could not be createdz!Submission created successfully. rr attachmentz %Y%m%d-%H%M%S_rr z/contents/attachments/rrz 2022-11-28)rrzX-GitHub-Api-Versionutf-8z Add file )rrrr)rrrZ download_urlz![Issue Attachment]()rrzFile attachment unsuccessful. rzFile attachment successful. zFile upload unsuccessful. zSPARC Reported Issue Submissionrr+z"SPARC Reported Feedback Submissionz,Confirmation email sent to user successful. .Confirmation email sent to user unsuccessful. )rrr"r))r%formrappr\rr$r#stripr&rFZGITHUB_ISSUE_ASSIGNEESrr|r&rfilenamer rxstrftimeruuid4hexr#r$r%rrrrputrrr6r@rr?rrrr) r4recaptcha_token task_typerPZ issue_bodyrZissueZ issue_urlr!r"r}Zresponse_messagerZresponse_statusr&r.Z file_contentr timestampZ unique_idZunique_filenamerrrrrr'Z image_urlZ comment_bodyrp email_body html_bodyrfrfrg create_issues                  rAc Cstjd}ddd|dgigdgdd}dd tjd }tj|||d }|}d}|d r:|d d d}|Stjd} d|||di} tj| || d } | js_td| j d| j | d}|S)Nz/objects/contacts/searchfiltersrZEQ)Z propertyNameoperatorrer^)Z filterGroups propertiesryrBearer rrr1rtrrlz/objects/contactsrD)r firstnamelastnamez!Hubspot contact creation failed:  ) rFHUBSPOT_V3_APIHUBSPOT_API_TOKENrr6rrrrrr ) rrGrHZ search_urlZ search_bodyrZsearch_resultsZ search_data contact_idZcreate_contact_urlZ contact_bodyZ create_resrfrfrgget_hubspot_contact3sB     rMc Cslddtjd}tjd}d||||di}tj|||d}|js.td|jd |j| d }|S) NrrErFz/objects/dealsrD)ZdealnameZ dealstagepipelineZlead_source_in_dealr1zHubspot deal creation failed: rIrl) rFrKrJrr6rrrr r) rZstagerNZ lead_sourcerZcreate_deal_urlZ deal_bodyZdeal_resdeal_idrfrfrgcreate_hubspot_deal_s    rPc Csddtjd}tjd}ttd}d||di}tj|||d}|j s6t d |j d |j | d }tjd |d |d} tj| |d} | j s^t d| j d | j tjd |d|d} tj| |d} | j st d| j d | j |S)NrrErFz/objects/notesirD)Z hs_note_body hs_timestampr1zHubSpot note creation failed: rIrlz/objects/notes/z/associations/deals/z /note_to_dealrz"Failed to associate note to deal: /associations/contacts/z/note_to_contactz%Failed to associate note to contact: )rFrKrJrr utcnowr>rr6rrrr rr;) rrOrLrZnote_urlrQZ note_payloadZnote_resnote_idZassociate_note_to_deal_urlZassociate_res_dealZassociate_note_to_contact_urlZassociate_res_contactrfrfrgcreate_hubspot_notets.  rUcCs\ddtjd}tjd|d|d}tj||d}|js*td|jd |j| S) NrrErFz/objects/deals/rRz/deal_to_contactrz,HubSpot deal to contact association failed: rI) rFrKrJrr;rrrr r)rOrLrZ associate_urlZ assoc_resrfrfrg#associate_hubspot_deal_with_contactsrVz/submit_data_inquiryc Cstj}tjd}tjds|rt|stddidfS|dd}|dd}|d d}|d d}|d d }|d }|d}|dd } |r^|r^|r^|r^|sftddidfS|dvrutdd|idfSd} d} d} |dkrtj ntj } |dkrtj ntj }|dkrtj nd}i}zt|||} Wnty}ztdt|ddfWYd}~Sd}~wwz t||| |} Wnty}ztd| t|ddfWYd}~Sd}~wwzt| | Wn ty}ztd| | t|ddfWYd}~Sd}~wwzt|| | } Wnty4}zd| | t|d}WYd}~nd}~wwdd| | | d }|rd!}d}| dkrRt||d"}n|dkr_t||d"nt||d"}t|}z"ttj||||d#dd$|d#<|r|d%dd$|d%<Wn6ty}z)|r|d%dd&|d%<|d'dt||d'<n d(| | t|d}WYd}~nd}~ww|r||t|d)fSt|d*fS)+Nr'r(r{r)rrr`rGrHrZ isAnbcFormrrPrZ isServiceFormz4Missing title, body, email, first name, or last name)researchZinterestr-rWrz&Failed to create or retrieve contact. )r{detailsrzFailed to create deal. )r{rLrXz'Failed to associate deal with contact. )r{rLrOrXz:Request successfully submitted, but note creation failed. )rrLrOrXz Request successfully submitted. r)rrrLrOrT"SPARC Form Submission Confirmationrrrz.Confirmation email sent to user successfully. rr3rXzQRequest successfully submitted, but confirmation email sent to user unsuccessful.r)r%r4rr5r\rr$r6rFZHUBSPOT_ONBOARDING_PIPELINE_IDZ HUBSPOT_GRANT_SEEKER_PIPELINE_IDZ,HUBSPOT_ONBOARDING_PIPELINE_INITIAL_STAGE_IDZ.HUBSPOT_GRANT_SEEKER_PIPELINE_INITIAL_STAGE_IDZANBC_LEAD_SOURCErMrr|rPrVrUrCrrBrArrrrupdate)r4r<rrGrHr=Z is_anbc_formrPrZis_service_formrLrOrTZ deal_pipelineZ deal_stageZdeal_lead_sourceZpartial_successr}rrpr?r@rfrfrgsubmit_data_inquirys      *    r]z/tasksc Cstj}d|vrCz#tjtjtj|ddd}|}d|vs"|ds)ddidfWSWn#tyB}z t d|WYd}~nd}~wwt j d sNddidfSd }t }d}d tjvrrd }tjd }t}t||t|t|j}t} |d } |r|r|dr| d|d7} t| |dddddd| grd|vr|dr|d} |dp| } d} t| | d}d}|rd|vr|d}|dkrd} n|dkrd} n |dkrd} n|dkrd} t| dkr| r|rttj| | |d d!d"|r|id#fSdid#fSdd$id%fS)&Nr'r r rr{r rrr(Fr.TrZ webViewLinkz Attachment: rPZ userEmailZ firstNamezSPARC SubmissionrZr`rnewszSPARC News SubmissioneventzSPARC Event SubmissionZtoolsAndResourceszSPARC Tool/Resource SubmissionZcommunitySpotlightzSPARC Story Submissionr z
attachment_filenamerzFailed registering user datar)r%r4rr6rFrrrrrr{r5r\rr9r&rrr|rVr7rrrArrrrrr!)r4rrrZhas_attachmentrrfileZ drive_clientclientrZ user_emailrrprr=rfrfrgreport_form_submission sn       rdz#/hubspot_contact_properties/c Cs4tjd|d}ddtjd}zDtj||d}|jdkr%t|WS|jdkr8td d |d d dfWStd d|jd|jddkrL|n|j d|jfWStj yx}ztdd|dt |ddfWYd}~Sd}~wt y}ztdd|dt |ddfWYd}~Sd}~ww)Nz/objects/contacts/zZ?archived=false&idProperty=email&properties=firstname,lastname,email,newsletter,event_namerrErFrrrzzContact not foundzNo contact with the email 'z' was found in HubSpot.r}zFailed to fetch contactz'HubSpot API responded with status code .r)r{rrXrz"Could not get contact with email 'z' due to a request error.rzInternal Server ErrorzDAn unexpected error occurred while fetching the contact with email 'z'.) rFrJrKrrrr$rrr rr|r)rrrrrrfrfrgget_hubspot_contact_propertiesGsR      rfz/subscribe_to_newsletterc Cstj}|d}|d}|d}|stddidfSd}z$t|\}}|dkr2|d d d}ntd |td |Wnty_}ztd |d|WYd}~nd}~wwg}t|t rs| d}t t d|}d|vr|| dd|||d|d|ddgi} tjd} ddtjd} ztj| | | d} | t| dfWStjjy}ztdt |idfWYd}~Sd}~ww)N email_address first_name last_namer{zEmail is requiredrr`rrD newsletterz"Unexpected response from HubSpot: zUnexpected error: z4Error while retrieving contact properties for email : ;Z NewsletterZinputs)rrGrHrjr)rDrlZ idPropertyz/objects/contacts/batch/upsertrrErFrr)r%rrr$rfrr{r isinstancer|splitlistfilterr]joinrFrJrKrr6rrr) rrrhriZnewsletter_propertyZcontact_propertiesrr}Zcurrent_newsletter_valuesrwrrrrfrfrgsubscribe_to_newsletterps^    "     rrc Cstjjtjd}z|jjjjt |gddd}Wnt y4}zt dd|dWYd}~Sd}~ww|s=t dddS|j sFt dd dS|j d sRt dd dS|j d d j}|j d igd }|rj|jnd}|j digd }|r{|jnd}|j d} t| d kr| d } | r| jdng} |j d} t| d kr| d } | r| jdng} dd| | D} |||| dS)N)Z access_token)rGrHrrj event_nameF)rLproperties_with_historyZarchivedrz2Exception thrown when getting contact properties: rz-Failed to retrieve contact data from HubSpot.zContact properties not foundrz Contact Email property not foundrrGr`rHrjrlrscSsg|]}|r|qSrfrfrtagrfrfrgrz*get_contact_properties..)rrGrHtags)hubspotZClientcreaterFrKZcrmZcontactsZ basic_apiZ get_by_idr|rr#rtrrerrn) object_idrc contact_datar}rZfirstname_datarGZ lastname_datarHZnewsletter_tags_dataZnewsletter_tagsZevents_tags_dataZ events_tagsrxrfrfrgget_contact_propertiessD         r}c Csd|d}ddtjd}|||d||d}z$tj|||d} t| jd kr8td |d | jd | j| WSt yi} z!td |d|| t dd|d|d| dWYd} ~ Sd} ~ ww)Nz#https://api.emailoctopus.com/lists/z /contactsrrErF)Z FirstNameZLastName)rgrBrrxrZ200z:Emailoctopus contact did not get added/updated for email: z. Returned a response of rkz4Could not add or update contact with email address: z in emailoctopus list: rz1Could not add/update contact with email address: z! from emailoctopus list with ID:  due to the following error: r) rFZEMAIL_OCTOPUS_API_KEYrr;r|rrr{r rrr#) Zlist_idrrGrHrxrrrrwrrrfrfrg"add_or_update_emailoctopus_contacts&   *rz/hubspot_webhookc Csd}ztjdd}Wn"ty,}ztd|tddidfWYd}~Sd}~wwt|tr4|sDtd|tddidfStj d ttj d |d tj vs`d tj vrutd tj tdd tj idfStj d }tj d }zt |}Wntytd|tddidfYSwzft t}||dkrtd|d|tddidfWStj}d}tj|dd}||||} ttjd| dtj} t| d} t| |stdtddidfWSWn&ty,} ztd| tdd| idfWYd} ~ Sd} ~ wwdd} |D]}t|tsEt d|q3t!"| |q3tdd d!d"fS)#NT)forcezInvalid JSON body: r{zInvalid JSON formatrz%Expected an array of webhook events: zExpected a non-empty JSON arrayz"Received Hubspot webhook request: zHubspot webhook request body: zX-HubSpot-Request-TimestampzX-HubSpot-Signature-V3zKRequired signature header(s) not present in the following request headers: z$Invalid signature timestamp format: z"Invalid signature timestamp formatr[z=z:hubspot_webhook..process_event..cSrFrfrurfrfrgr>rzUnsupported subscription type: zError processing event rk) r5Z app_contextrrr{r}rrrFZEMAIL_OCTOPUS_MASTER_LIST_IDr]r\) r_Zsubscription_typer{r|rrGrHrZemailoctopus_contactZ tags_to_addrvZtags_to_removeZupdated_contact_tagsrfrfrg process_event sZ            """z&hubspot_webhook..process_eventzSkipping non-dict event: rz/Webhook request received and signature verifiedrr)#r%rrrr{r$rmror5loggerrrrrrtimerrrhmacnewrFHUBSPOT_CLIENT_SECRETencodehashlibsha256digestrrrZcompare_digestdictrexecutorsubmit)rr}Zsignature_headerZtimestamp_headerZsignature_timestamp current_timerrZstringified_bodyZ raw_stringZhashed_signatureZbase64_hashed_signaturerrr_rfrfrghubspot_webhooksr        "% rz/get-organ-curies/c Cstjd}t|}i}ztjtjdtj|d}t | }Wt|St yD}zt d|ddddfWYd}~Sd}~ww) Nrqr4r5zFailed getting Uberon IDsr~ BaseExceptionrr)r%rrAr6rr6rFr7r8rOrrrr{r$)rqZ requestBodyrmrrrfrfrgget_available_uberonidsOs&  rz/get-related-terms/c Cstjjdddtjjdddtjjdddtjd}i}ztjtjd ||d }t|}Wt |St yV}zt d ||d d ddfWYd}~Sd}~ww)N directionZOUTGOING)rrelationshipTypez BFO:0000050entailr)rrrrz/graph/neighbors/)rz*Failed getting related terms with payload r~rrr) r%rrrFr8rZSCI_CRUNCH_SCIGRAPH_HOSTrQrrrr{r$)rFrwrmrrrfrfrgget_related_termsgs,rz /simulation_ui_file/cCsttt|}t|j}z2|dd}|d}|dddd}tdd|d |}td d|}ttt ||WSt yPt d d d YdSw)NrtrZs3urizabi-simulation-filerYrz s3://[^/]*/r`zfiles/z s3://|/.*rzz$no simulation UI file could be foundr) rLrDr4rrrresubr$r2rr#)rTrtZ results_jsonrer#rrZs3_bucket_namerfrfrgsimulation_ui_files   rz /pmr_filecCszt}|r5d|vr5zttjd|d}|jdkr$t|j WS| WSt dddYdSt ddddS)Nrr rrz invalid pathrz missing path) r%rrr6rFZPMR_HOSTrrrrrr#)rrrfrfrgpmr_files   rz/start_simulationcCsJt}|rd|vrd|dvrd|dvrtt|StddddS)Nsolverrrrz)Missing solver name and/or solver versionr)r%rrrdo_start_simulationr#rrfrfrgrWs$rWz/check_simulationcCsRt}|r!d|vr!d|vr!d|dvr!d|dvr!tt|StddddS)NZjob_idrrrrz1Missing solver name, solver version and/or job idr)r%rrrdo_check_simulationr#rrfrfrgrXs,rXz/pmr_latest_exposurecCst}|rKd|vrKz4tj|dddid}|jdkr:z|dddd dd }Wnd }Yt|d WS|WStd ddYdStd dddS)NZ workspace_urlrz$application/vnd.physiome.pmr2.json.1rr collectionitemsrZlinksZhrefr`rrzInvalid workspace URLrzMissing workspace URL)r%rrrrrr$r#)rrrrfrfrgpmr_latest_exposures$   $ rz/onto_term_lookupc Cstjd}ddi}dtji}t|}z3tjtjd|||d}|}|dd}|dd}|d kr@|d }|d } | WSd d i} | WSt ya} zt d| WYd} ~ t dSd} ~ ww)Nr|rrrrvrrrrgtotalr^rrhlabel not found.An error occured while fetching from SciCrunchr) r%rrrFr8r7rZSCI_CRUNCH_INTERLEX_HOSTrrrr{r#) r|rrrFrrtrgrrmrrrfrfrgfind_by_onto_terms,   rz/dataset_citations/c Csddi}dtji}t|}z5tjtjd|||d}|}|dd}|ddd}|d kr<|d }|d } | WSd d i} | WStyZ} z t d| WYd} ~ nd} ~ wwt dd|didfS)Nrrrrvrrgrrer^rrhz dataset idrrrz:An error occured while fetching citation info for dataset z from SciCrunchr) rFr8r=rrSCI_CRUNCH_CITATIONS_HOSTrrrr{r$) rrrrFrrtrgrrmrrrfrfrgget_dataset_citationss* rz/total_dataset_citationsc Csddi}dtji}dddiidddd iiid }z+tjtjd |||d }|}|d dd}tdd|D}td|idfWSty[}z t d|WYd}~nd}~wwtddddfS)NrrrrZ match_allZ Citationsrr=zcitations.type)r?r@rFrrvrrrscss|]}|dVqdS)Z doc_countNrf)rrrfrfrgr( sz.get_total_dataset_citations..total_citationsrz>An error occured while fetching total citations from SciCrunchr)rrr) rFr8rrrrsumr$rrr{)rrrFrrtrsrrrfrfrgget_total_dataset_citations s2 rz/search-readme/c Cstd|}ddtji}z tj||d}|WStjjy9}zt|t |dddfWYd}~Sd}~ww)Nz2https://dash.readme.com/api/v1/docs/search?search=rzBasic )rrz9Readme is not currently reachable, please try again laterr}r) rFZREADME_API_KEYrr6rrrrr{r|)rFrrrrrfrfrg search_readme. s"  rz/metricscCstSrb)rrfrfrfrgmetricsA srz/event_updatedcCsrtjd}|tjkrtdd|ddSt}|r1zt|WStdd|dYdStddddS)NZevent_updated_secret_keyr+zInvalid secret key: rrzInvalid event data: zMissing event data)r%rrrFZCTF_CDA_ACCESS_TOKENr#rr)Z secret_keyr_rfrfrg event_updatedG s   riQz/all_dataset_idscC"t}dd|D}d}||S)NcSg|]}t|qSrfr|relementrfrfrgr[ rwz#all_dataset_ids.., )r rqroZ string_list delimiterrfrfrgall_dataset_idsW  rz/all_dataset_uuidscCr)NcSrrfrrrfrfrgrc rwz%all_dataset_uuids..r)r rqrrfrfrgall_dataset_uuids_ rrz/total_protocol_viewscCsFtt}|dus |dstddddfS|d}td|idfS)NrzTotal views not yet calculated.) total_viewsrrr)rprotocolMetricsTablerr$)Z table_staterrfrfrgget_total_protocol_viewsg s rz/contact_supportc Cst}|d}|d}|d}|dd}|r|r|s'tddidfS|rVzttj||t d|itj Wnt yU}ztdd id fWYd}~Sd}~wwtdd id fS) NrrrrprYr{zMissing required fieldsrz-Confirmation email sent to user unsuccessful.rzMessage received successfully.r) r%rrr$rrrFrr?rZSERVICES_EMAILr)rrrrrpr}rfrfrgcontact_supportt s(      r)rxrr)NNrb(1atexitZapp.metrics.pennsieverZapp.metrics.contentfulrrrZ!scripts.update_contentful_entriesrrZapp.metrics.algoliarr r r r Zapp.metrics.gar rrrrrZ"scripts.update_featured_dataset_idrrZscripts.update_protocol_metricsrrZapp.osparc.servicesrr/rZboto3rrrrryZhubspot.crm.contactsrrrrrr urllib.parserZ!apscheduler.schedulers.backgroundrZapscheduler.triggers.combiningrZapscheduler.triggers.daterZapscheduler.triggers.intervalrZbotocore.exceptionsrconcurrent.futuresrr r!Zflaskr"r#r$r%Z flask_corsr&Zflask_marshmallowr'rir(Zpennsieve2.directr)Zpennsieve.baser*rZPILr+Z requests.authr,Z flask_cachingr-Zapp.scicrunch_requestsr.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=Zscripts.email_senderr>r?r@rArBrC threadingrDZ xml.etreerE app.configrFZ app.dbtablerGrHrIrJrKZapp.scicrunch_process_resultsrLrMrNrOrPrQrRZapp.serializerrSZ app.utilitiesrTrUrVZapp.osparc.osparcrWrrXrZapp.biolucida_process_resultsrrYr basicConfigrur5Z LOG_LEVELupper log_levelrsetLevelgetattrWARNINGrZ DEPLOY_ENVr\cacheZmarrrcZSPARC_PORTAL_AWS_KEYZSPARC_PORTAL_AWS_SECRETr]rZ DATABASE_URLZdb_url startswithr!rAttributeErrorrrrrobjectr_Z errorhandlerr~Zbefore_first_requestrrrrrrrrZprotocol_metricsrrrrrZannotation_cleanup_schedulerZadd_jobZremoveExpiredStaterrrrrrrrrZfeatured_dataset_id_triggerrregisterZrouterrrZDEFAULT_S3_BUCKET_NAMErrrrr%r)r0r2r:r<rCrGrJrMrOrQrVrWrXr^rkrarnrrrurDr{rrrrrrrrrrrrcachedrrrrrrrrrrrrrrrrrrrrr r rr&rArMrPrUrVr]rdrfrrr}rrrrrrrrrrrrrrrrrrfrfrfrgs                   H    $              &               >  %            .   ,    0           :   $      g," g;(;' e