a Eg @s ddlZddlZddlmZddlmZmZmZddlm Z m Z ddl m Z m Z mZmZddlmZmZddlmZddlmZmZdd lmZddlZddlZddlZddlZddlZddlZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'ddl(m)Z)ddl*m+Z+ddl,m,Z,m-Z-ddl.m/Z/m0Z0m1Z1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8ddl9m:Z;ddlm?Z?ddl@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQddlRmSZSmTZTmUZUmVZVmWZWmXZXddlYmZZZddl[m\Z\ddl]m^Z^ddl_m`Z`maZambZbddlcmdZdmeZemfZfmgZgmhZhmiZimjZjdd lkmlZldd!lmmnZnmoZodd"lpmqZrdd#lpmsZtdd$lumdZvmwZxeye/ezZ{e^j|e{j}d%<eAe{d&d'd(d)Z~e4e{e6e{ZeSZdaejd*e^je^jd+d,ZeZZze`e^jZWneydZYn0zeae^jZWney:dZYn0zebe^jZWneybdZYn0Gd-d.d.eZe{d/d0d1Ze{jd2d3Ze#Ze#Ze#Ze#Ze#Zejsed4ee^j|d5kre#ZeZeeejejd6d7d7d8d9ddd:e^j|d;krTe^jdd?d@iZe{jdAdBZiZeZe ZeZe{jdCdDZeZe{jdEdFZejedGdHdIejedGdHdIejedGdJdKe%e'e)dHdLgZedMdNedOdPZeee{dQdRdSZe{jdTdUgdVdWdXZdYdZZe{d[d\e^jfd]d^Ze{d_d`daZe{dbe^jfdcddZe{dee^jfdfdgZdhdiZe{djdkdlZdmdnZe{doe^jfdpdqZe{drdsdtZe{due{dvdwdxZe{dydzd{Ze{d|d}d~Ze{de{dddZe{de{dddZe{de{dddZe{dddZe{dddZe{dddZe{dddZe{de^jfddZe{dddZddZe{jdddddde{dd'ddZe{jdddide{dddZe{dddZddZddZddZe{dddZe{dddZe{dddZe{dddZe{dddÄZe{jddgdVddDŽZe{jddgdVddʄZe{jddgdVe~jd'd̍dd΄Ze{jddgdVddфZe{jddgdVddԄZe{jddgdVd(dd؄Ze{jddgdVddۄZe{jddgdVddބZe{jddgdVddZe{jddgdVddZddZddZddZe{jddUgdVddZe{jddUgdVddZe{jddUgdVddZe{jddUgdVddZe{jddUgdVddZe{jddUgdVddZe{jddUgdVddZe{jddgdVddZe{dddZe{dddZe{d d d Ze{jd dUgdVd dZe{jddUgdVddZqe{jddUgdVddZse{jddUgdVddZe{dddZe{jddgdVddZe{jddgdVdd Ze{jd!dUgdVd"d#Ze{jd$dgdVd%d&ZdS()N)get_download_count)init_cf_cda_clientget_funded_projects_countget_featured_datasets)update_all_events_sort_orderupdate_event_sort_order)get_dataset_countinit_algolia_clientget_all_dataset_idsget_associated_datasets)init_ga_reportingget_ga_1year_sessions) MonthlyStats)set_featured_dataset_id#get_featured_dataset_id_table_state)OSparcServices)urlparse)BackgroundScheduler) OrTrigger) DateTrigger)IntervalTrigger) ClientError)datetime timedelta)Flaskabortjsonifyrequest)CORS) Marshmallow) Pennsieve)UnauthorizedException)Image) HTTPBasicAuth)Cache)create_doi_querycreate_filter_requestcreate_facet_querycreate_doi_aggregatecreate_title_querycreate_identifier_query!create_pennsieve_identifier_querycreate_field_querycreate_request_body_for_curiescreate_onto_term_querycreate_multiple_doi_query create_multiple_discoverId_querycreate_anatomy_queryget_body_scaffold_dataset_idcreate_multiple_mimetype_query) EmailSenderfeedback_emailgeneral_interest_emailissue_reporting_email#creation_request_confirmation_emailservice_interest_email)Lock) ElementTree)Config)MapTable ScaffoldTableFeaturedDatasetIdSelectorTable)process_resultsprocess_get_first_scaffold_inforeform_aggregation_resultsreform_curies_resultsreform_dataset_resultsreform_related_termsreform_anatomy_results)ContactRequestSchema)img_to_base64_strget_path_from_mangled_list)start_simulation)check_simulation)r@process_resultZENVZ SimpleCachei,)Z CACHE_TYPEZCACHE_DEFAULT_TIMEOUT)configs3z us-east-1)Zaws_access_key_idZaws_secret_access_keyZ region_namec@s`eZdZdZeedZdZe ddZ ddZ e dd Z d d Z e d d ZddZdS) Biolucidai?BFcCs |t_dSN)rO_tokenvaluerU3/home/cmiss/Jenkins/workspace/SPARC-API/app/main.py set_tokenaszBiolucida.set_tokencCs|jSrQ)rRselfrUrUrVtokeneszBiolucida.tokencCs |t_dSrQ)rO _expiry_daterSrUrUrVset_expiry_datehszBiolucida.set_expiry_datecCs|jSrQ)r[rXrUrUrV expiry_datelszBiolucida.expiry_datecCs |t_dSrQ)rO_pending_authenticationrSrUrUrVset_pending_authenticationosz$Biolucida.set_pending_authenticationcCs|jSrQ)r^rXrUrUrVpending_authenticationssz Biolucida.pending_authenticationN)__name__ __module__ __qualname__rRrnowrr[r^ staticmethodrWrZr\r]r_r`rUrUrUrVrO\s   rOcCstt|ddfS)N)errorrf)rstr)erUrUrVresource_not_foundwsrjc CszttjtjdtjdaWntjjyV}z t dt |WYd}~nvd}~0t y}z t dt |WYd}~n@d}~0t y}z t dt |WYd}~n d}~00dS)NF)Z api_tokenZ api_secretZ env_overridehostz#Unable to connect to Pennsieve hostz&Unable to authorise with Pennsieve Apiz Unknown Error) r r<ZPENNSIEVE_API_TOKENZPENNSIEVE_API_SECRETZPENNSIEVE_API_HOSTpsrequests exceptions HTTPErrorloggingrgPSUnauthorizedException Exception)errrUrUrVconnect_to_pennsieve|s     rtz6Starting scheduler for featured dataset id acquisition productionZcron*12)yearmonthdayhourminutesecondZ developmentZFALSEz8Starting scheduler for updating contentful event entriesz US/Eastern)r|timezonec CstdzLtjtjdd}|jrTd|jddvrT|}t |d}|t d<Wn0t y}zt d |WYd}~n d}~00t jstd t dS) NzGetting oSPARC viewersz/viewersurlapplication/json content-typerPdata file_viewersz!Could not retreive oSPARC viewersz1Starting scheduler for oSPARC viewers acquisition)rpinformgetr<OSPARC_API_HOSTokheadersjsonbuild_filetypes_table osparc_datarrrgviewers_schedulerrunningstart)reqZviewerstablerirUrUrVget_osparc_file_viewerss   " rcCsptdtrtt}|td<tr2tt}|td<trFtt}|td<t }|td<t j sltdt dS)NzGathering metrics dataZ1year_sessions_countZ dataset_countZfunded_projects_countZ1year_download_countz*Starting scheduler for metrics acquisition) rprgoogle_analyticsr usage_metricsalgoliar contentfulrrmetrics_schedulerrr)Z ga_responseZalgolia_responseZ cf_responseZ ps_responserUrUrV get_metricss  rc Csntdz.tjtjdd}|}t|dWn0t yh}zt d|WYd}~n d}~00dS)NzFetching oSPARC servicesz /servicesrrz%Request to get oSPARC services failed) rprrmrr<rrosparc_servicesZ set_servicesrrrg)rZ services_resprirUrUrV get_servicess rinterval)functriggerdays)rrhours)rcCsttSrQ)rfeaturedDatasetIdSelectorTablerUrUrUrVrcCs|tdtjrttdtjr0ttdtjrHttdtjr`ttdtjrxtdS)Nz1Stopping scheduler for oSPARC viewers acquisitionz*Stopping scheduler for metrics acquisitionz2Stopping scheduler for updating contentful entriesz3Stopping scheduler for updating featured dataset idz&Stopping scheduler for oSPARC services) rprrrshutdownr)update_contentful_event_entries_schedulerfeatured_dataset_id_schedulerservices_schedulerrUrUrUrVshutdown_schedulerss     rz/healthcCstddiS)NstatusZhealthy)rdumpsrUrUrUrVhealthsrz/contactPOST)methodsc Cshttj}t|}|d}|d}|d}t|||tt j |dt d|it ddiS)NnameemailmessagezFeedback submissionrsent)rloadsrrrGload email_senderZ send_emailsendgrid_emailr< SES_SENDERr5 substituter)rZcontact_requestrrrrUrUrVcontacts  rcCstjd||d|d|d}|S)N get_object requester)BucketKey RequestPayerZResponseContentType)ZParamsZ ExpiresIn)rNZgenerate_presigned_url) s3BucketNamekey content_type expirationresponserUrUrVcreate_s3_presigned_url#s  rz /downloadicCs6tjd}tjd|}tjdd}t||||S)NrrZ contentTypezapplication/octet-stream)rargsrr)r bucket_namerrrrUrUrVcreate_presigned_url.s rz/thumbnail/neurolucidac Cstj}d|vsd|vsd|vr*tdddStjd}zPtj||dd }||jd krz|j d d d krzt |j WStddWn|tj jytdddYStj jytddYStj jy}z tddt|WYd}~Sd}~00dS)NversionZ datasetIdpathQuery arguments are not valid. descriptionz /thumbnail)paramstimeoutz Content-Typeunknownz image/pngzFailed to retrieve thumbnail.z0Unable to make a connection to NEUROLUCIDA_HOST.iz&Request to NEUROLUCIDA_HOST timed out.z)Error while requesting NEUROLUCIDA_HOST: )rrrr<ZNEUROLUCIDA_HOSTrmrZraise_for_status status_coderbase64 b64encodecontentrnConnectionErrorZTimeoutZRequestExceptionrh) query_argsrrrirUrUrVthumbnail_from_neurolucida_file7s"   rz/thumbnail/segmentationc Cstj}d|vrtdddS|d|}|d}d}d}d}d}d }|} |rP|sTz"tj||d |d | d d } Wnvty} z^| jdddkrtdd|ddWYd} ~ Stdd|ddWYd} ~ SWYd} ~ n d} ~ 00| d d}d|v}d|v}|r"|s"| |7} n||7}| |7} t ||krFtdd|ddSqF|durrtdd|ddS|| dd} | d| dd} t | }|j}d}|D]}||jdd7}qt|}tdt|dt|df|}t|}|S)z Extract a thumbnail from a mbf xml file. First phase is to find the thumbnail element in the xml document. Second phase is to convert the xml to a base64 png. rrrrrNFrizbytes=-r)rrZRangerErrorCode NoSuchKeyrfCould not find file: ''Unknown error for file: 'BodyzUTF-8z z#Could not find thumbnail in file: 'rPrZRGBZrowscols)rrrrrNrrrreaddecodelenfindr; fromstringattribtextbytesfromhexr" frombytesintrH)rrrrresourceZstart_tag_foundZ end_tag_foundZ start_byteoffsetZend_byterexZstart_thumbnail_elementZ thumbnail_xmlxmlZ size_infoZim_datachildZ byte_im_dataZimZ base64_formrUrUrVextract_thumbnail_from_xml_fileNsZ     6      "rz/exists/cCsftj}|d|}ztj||dd}Wnty@ddiYS0|dd}|dkr^ddiSddiS) Nrrrrrexistsfalse ContentLengthrtrue)rrrrN head_objectr)rrrr head_responsecontent_lengthrUrUrV url_existss    rcCs0|dd}|d|d}t|}|S)Nzhttps://api.pennsieve.io/z"https://api.pennsieve.io/discover//)replacerfindrmrr)urirrUrUrVfetch_discover_file_informations  rz/s3-resource/discover_pathc Cstjd}z6t|}d|vr@|ddkr@|dd}|dWSWn0tyr}ztd|WYd}~n d}~00tdd |d S) NrZ totalCountrfilesrrzFailed to retrieve uri {uri}rfzFailed to retrieve uri r)rrrrrrrprgr)rZ json_response file_inforrUrUrVget_discover_paths  "rc CszDtj||dd}|dtj}|rB|tjksBtdd|dWSWntjjy}zz|j ddd krzdWYd}~S|j ddd krdWYd}~St|j dd|j ddWYd}~SWYd}~nd}~00dSdS)NrrrizFile too big to download: rrrZ404rf.Provided path was not found on the s3 resourceZ403EThere is a permission issue when accessing the file at specified pathMessage)rOK)rfr )r r ) rNrrr<ZDIRECT_DOWNLOAD_LIMITrbotocorernrr)rrrrrsrUrUrVs3_header_checks @rz/s3-resource/c Cstj}|d|}|}t||}|ddks8|ddkrt|}||krTtdddt||}|ddkrp|}n2|ddkrtdddn|ddkrtdddtj||d d }tjd }|d } |durt | S| S) Nrrrfr r rrr rrZ encodeBase64r) rrrrrIrrNrrrr) rrrrZs3_pathrZs3_path_modifiedZ response2 encode_base64rrUrUrVdirect_download_urls2           rz /scicrunch-dataset//c Cs|ddd|}t|}z&tjtjdtj|d}|WStjj y}z$t |t d|iWYd}~Sd}~00dS)NzDOI:rPr/_search?api_key=rrg) rr%rmpostr<SCI_CRUNCH_HOSTKNOWLEDGEBASE_KEYrrnrorprgr)Zdoi1Zdoi2doirrrsrUrUrVsci_dois  rz /pubmed/z/pubmed//c Cs`ztd|d}|jWStjjyZ}z$t|td|iWYd}~Sd}~00dS)Nz https://pubmed.ncbi.nlm.nih.gov/rrg) rmrrrnrorprgrr)id_rrsrUrUrVpubmed s  rz/scicrunch-query-string/c Cstjd}tjd}tjd}tjd}t||||}z*tjtjdtj |d}t | WStj j y}z$t|t d|iWYd}~Sd}~00dS)Nfieldcuriesizefromrrrg)rrgetlistrr,rmrr<rrr@rrnrorprgr)fieldsrrZfrom_rrrsrUrUrV sci_organs     r!z/dataset_info/using_doicCs<tjd}tjd}t|}|dur4tt|St|S)NrZ raw_response)rrrr%rDdataset_search)rrawqueryrUrUrVget_dataset_info_doi*s    r%z!/dataset_info/using_multiple_doisz"/dataset_info/using_multiple_dois/cCs tjd}t|}tt|S)Ndois)rrrr/r@r")r&r$rUrUrVget_dataset_info_dois5s r'z./multiple_dataset_info/using_multiple_mimetypez//multiple_dataset_info/using_multiple_mimetype/cCs tjd}t|}tt|S)Nq)rrrr3r@r")r(r$rUrUrVget_file_info_from_mimetype=s r)z(/dataset_info/using_multiple_discoverIdsz)/dataset_info/using_multiple_discoverIds/cCs tjd}t|}tt|S)N discoverIds)rrrr0r@r")r*r$rUrUrVget_dataset_info_discoverIdsGs r+z/dataset_info/using_titlecCs tjd}t|}tt|S)Ntitle)rrrr)rDr")r,r$rUrUrVget_dataset_info_titlePs r-z%/dataset_info/using_object_identifiercCs tjd}t|}tt|SN identifier)rrrr*rDr"r/r$rUrUrV"get_dataset_info_object_identifierXs r1z/dataset_info/anatomycCs6tjdd}|dkr"tdddSt|}tt|S)Nr/rfz Identifier for API call not set.r)rrrrr1rFr"r0rUrUrVget_dataset_info_anatomy`s  r3z(/dataset_info/using_pennsieve_identifiercCs tjd}t|}tt|Sr.)rrrr+rDr"r0rUrUrV%get_dataset_info_pennsieve_identifierks r4z/segmentation_info/c CsZtj}d|vrtdddS|d|}|d}z6t||}|ddkrTtdddtj||d d }Wnvty}z^|jd d d krtdd|ddWYd}~Stdd|ddWYd}~SWYd}~n d}~00|d }t |}| d}i} |dur|j | d<nddddd| d<| d} | durJ| j | d<n ddi| d<| S)N dataset_pathrz,Query arguments must include 'dataset_path'.rrrrfr rrrrrrrrrz./{*}sparcdata/{*}subjectsubjectrP)ZageZsexspeciesZ subjectidz./{*}sparcdata/{*}atlasZatlasZorgan)rrrrrrNrrrrr;rrr) rrrr5rrrrZsubject_elementrZ atlas_elementrUrUrVget_segmentation_info_from_filess<        6         r8z/current_doi_listcCsBt}tt|}g}|ddD]}||ddq"d|iS)Nrbucketsrrresults)r(rBr"append)r$r:Z doi_resultsresultrUrUrV get_all_dois  r=c Cstz0|}dtji}tjtjd||d}|WStjjyn}z"t |t d|iWYd}~Sd}~00dS)Napi_key/_search)rrrg) r<rrmrrrrnrorprgr)r$payloadrrrsrUrUrVr"s  r"z/search/rP )r$limitr)defaultsz/search/c Csztjddurtjd}tjddur:tjd}tjddurVtjd}ttjd|d|d|dtj }t|WStj j y}z$t |t d|iWYd}~Sd}~00dS) NrBr$rz /_search?q=z&size=z&from=z &api_key=rg)rrrrmr<rrr@rrnrorprgr)r$rBrrrsrUrUrV kb_searchs   * rDz/filter-search/r$z/filter-search//c Cstjd}tjd}tjd}tjd}t|||||}z,tjtjdtj |d}t | }Wnltj j y}z,t|tt|ddd fWYd}~Sd}~0t jytd d d d fYS0|S) NtermZfacetrrrrzc Cst|\}}g}||D]}||d|dd<z2tjtjdtj|d}|}||Wqtjyt dddd fYSt y}zt d ||WYd}~qd}~00qg}|D]} || d|d 7}qt |S) NZ aggregationsrJrrrrGrHrIrz$Could not search SciCrunch for path r9) r'rmrr<rrrr;rHrrrrprg) type_Ztype_maprr:rrZ json_resultrrJr<rUrUrV get_facetss.   *rMcCs&d|vr"t|d}|j|d<dS)NZreadmeZmarkdown)rmrr)respZmark_reqrUrUrVinject_markdown srOc CsB|d}|d}|dus$|dur(dSt|}|j}ztj|d|dd}Wn|tytjdk}|rvt dztj|d|dd}Wn6ty}z|rt |WYd}~YdSd}~00Yn0|d  }zt |} Wn2ty}zt |WYd}~dSd}~00| d | d | d d |d<dS)Nidrz{}/files/template.jsonrrZTRUEzXRequired file template.json was not found under /files folder, trying under /packages...z{}/packages/template.jsonruuidrr)rQrrZstudy)rrnetlocrNrformatrr<SPARC_API_DEBUGGINGrpwarningrgrrr ValueError) rNrrZ parsed_uribucketr debuggingritemplateZ template_jsonrUrUrVinject_template_datasL          rZcCsFi}|D]8}|d}|d=||ds2g||<|||q|S)NZ file_typeF)lowerrr;)Zosparc_viewersrZviewerZfiletyperUrUrVr@s  rz/sim/dataset/c Cstjdkrz@tdtj|}|jrH|}t |t |t |WSWn6t y}zt d||WYd}~n d}~00tdddSdS)NGETz{}/datasets/{}Could not fetch SIM dataset rfResource not foundrrmethodrmrrSr<DISCOVER_API_HOSTrrrOrZrrrrprgr)rr json_datarrUrUrV sim_datasetJs (rcz&/sim/dataset//versions/c CstjdkrzBtdtj||}|jrJ|}t |t |t |WSWn<t y}z$t d|d||WYd}~n d}~00tdddSdS)Nr\z{}/datasets/{}/versions/{}r]z version rfr^rr_)rZversion_rrbrrUrUrVsim_dataset_versionsXs .rdz/get_osparc_datacCsttSrQ)rrrUrUrUrVget_osparc_datahsrez /sim/servicecCsTtjdkrPtjd}tjjddtd}tjjddtd}t|||}t|SdS)Nr\searchrBr)defaulttypeskipr)rr`rrrrZsearch_servicesr)rfrBrir:rUrUrV osparc_searchms   rjz /sim/filecCs tjdkrtj}td|iSdS)Nr\r)rr`rZfile_extensionsr) extensionsrUrUrVosparc_extensionsws  rlz/project/r\cCs4t|}t|ddkr$t|dStddddS)Nhitsrrfr^r)r rrr)Z project_iddatasetsrUrUrVdatasets_by_project_ids roz"/get_featured_datasets_identifierscCs dtiS)N identifiers)rrUrUrUrV!get_featured_datasets_identifierssrqz/get_featured_dataset)rc Csttd}|dkrd}zBtdtj|}|dgkrVtdtjd}|WSty}zt d||WYd}~n d}~00t ddd dS) Nfeatured_dataset_idr2 z{}/datasets?ids={}rnzCould not get featured dataset rfz,An error occured while fetching the resourcer) rrrmrrSr<rarrrrprgr)rrrrrUrUrVget_featured_datasets  (rtz/get_owner_email/csPtjj}tjj|}fdd|D}|s:tdddntd|djiSdS)Ncsg|]}|jkr|qSrU)Zint_id).0xowner_idrUrV rz#get_owner_email..rfzOwner not foundrrr)rlZ_apiZ _organizationZ organizationsZ get_membersrrr)rxZorgmembersresrUrwrVget_owner_emails r|z!/get_body_scaffold_info/cCs:t|}|r(t|}tt|}|r(|Stdd|dS)NrfzWhole body info not found for r)r2r+rAr"r)r7rPr$r<rUrUrVget_body_scaffold_infos r}z/thumbnail/Fc Cst}zt|stWdn1s00Ytjd|}d|i}tjd||d}t |j }|dkr|st| dWdn1s0Yt |d}|WSty}ztd||WYd}~n d}~00td d S) Nz/thumbnail/{0}rZr\rseyJzdGF0dXMiOiJBZG1pbiB1c2VyIGF1dGhlbnRpY2F0aW9uIHJlcXVpcmVkIHRvIHZpZXcvZWRpdCB1c2VyIGluZm8uIFlvdSBtYXkgbmVlZCB0byBsb2cgb3V0IGFuZCBsb2cgYmFjayBpbiB0byByZXZlcmlmeSB5b3VyIGNyZWRlbnRpYWxzLiJ9rPTz Could not get the thumbnail for rfz-An error occured while fetching the thumbnail)rObiolucida_lockrZauthenticate_biolucidar<BIOLUCIDA_ENDPOINTrSrmrrrrrWthumbnail_by_image_idrrrprgr)image_idZrecursive_callblrrrZencoded_contentrrUrUrVrs($ ( (rz/image/c Csltjd|}ztd|}t|WSty`}zt d||WYd}~n d}~00t ddS)Nz/image/info/{0}r\zCould not get image info for rfz/An error occured while getting the image's info) r<rrSrmrprocess_biolucida_resultrrrrprgr)rrrrrUrUrVimage_info_by_image_ids (rz/image_search/c Csjtjd|}ztd|}|WSty\}ztd||WYd}~n d}~00ddidfS)Nz%/imagemap/search_dataset/discover/{0}r\z$Could not search images for dataset rgz3An error occured while searching images for datasetrf) r<rrSrmrrrrrprg)Z dataset_idrrrrUrUrVimage_search_by_dataset_ids  (rz/image_xmp_info/cCsvtjd|}ztd|}Wn"tjjyBtdddYS0|}|ddkrdt |dStdd |dS) Nz/image/xmpmetadata/{0}r\r)Unable to make a connection to Biolucida.rrsuccessrzXMP info not found for ) r<rrSrmrrnrrrprocess_biolucida_resultsrrr<rrUrUrVimage_xmp_infos  rz/image_blv_link/cCsztjd|}ztd|}Wn"tjjyBtdddYS0|}|ddkrht d|diStdd |dS) Nz/image/blv_link/{0}r\rrrrrlinkzBLV link not found for ) r<rrSrmrrnrrrrrrUrUrVimage_blv_links rcCsdt}tjd}tjtjdd}g}i}tjd||||d}|jtjj kr`| }| |ddS)Nz /authenticaterP)usernamepasswordrZr)rrrrZ) rOr<rZBIOLUCIDA_USERNAMEZBIOLUCIDA_PASSWORDrmrrZcodesrrrW)rrr@rrrrrUrUrVrs rcCshd}tjdrd}|rXt}|rJd|vrJ|d}|||}td|iStdddn td d ddS) NTTESTINGFstaterQrzState not specifiedrrfDatabase not available)apprMrget_jsonZ pushStaterr)rZcommitrbrrQrUrUrVget_share_links    rcCs^|rNt}|r@d|vr@|d}||}|r@td||iStdddn tddddS)NrQrrz#Key missing or did not find a matchrrfr)rrZ pullStaterr)rrbrQrrUrUrVget_saved_state/s  rz/map/getshareidcCsttSrQ)rmaptablerUrUrUrVget_map_share_link=srz /map/getstatecCsttSrQ)rrrUrUrUrV get_map_stateCsrz/scaffold/getshareidcCsttSrQ)r scaffoldtablerUrUrUrVget_scaffold_share_linkIsrz/scaffold/getstatecCsttSrQ)rrrUrUrUrVget_scaffold_stateOsr/tasksc# CsJtj}d|vrzFtjtjtj|ddd}|}d|vsD|dsRddidfWSWqty}zt d|WYd}~qd}~00nt j d sddidfS|r:d |vr:d |vr:|d }|d }|d }d d tj i}dtjd}tjtjtjtjtjtjg} tjtjtjtjtjtjg} tj} d} d} g}|rNd|vrN|d} | dkrdtjd}tjtjtjg} tjtjtjg} tj} tj} | dkrdtjd}tjtjtjg} tjtjtjg} tj} tj} n| dkrdtjd}tjtjtjg} tjtjtjg} tj} tj} nt| dkrddtjd}tjtjtjg} tjtjtjg} tj} tj } n.| dkr| !tj"tj#g| !tj"tj#g| dkrd| }tj$||d}d|vr|dgkr|ddd |}|ddd}||| | | dddid}tj|||d}tj%}d|vr|dgkr|ddd }|rd!|vr|d!}|&}|j'}|j(}d tj ||d"d#}d|d$}ztj|||d%Wn.ty}zt)|WYd}~n d}~00|D]}d|}tj$||d}d|vr|dgkr|ddd |ddd |ddd&|ddd'|ddd(d|gddid)}tj|||dq|j*d*kr0d+|vr|d+rd,|vr|d,d-krd.} t+,d/|i}!| d0krd1} t-,d/|i}!n| d2kr4d3} t.,d/|i}!n| d4krRd5} t/,d/|i}!n| dkrpd6} t0,d/|i}!nv| dkrd7} t0,d/|i}!nX| dkrd8} t0,d/|i}!n:| dkrd9} t0,d/|i}!n| dkrd:} t0,d/|i}!|d+}"t1|"dkrt23tj4|d+| |!t5|||ddd d;S|Sn t6ddS)?NZ captcha_token)Zsecretr)rrrrgzFailed Captcha Validationiz0Could not validate captcha, bypassing validationrr,r AuthorizationzBearer z%https://www.wrike.com/api/v4/folders/rrPrhnewseventZtoolsAndResourcesZcommunitySpotlightZresearchz#https://www.wrike.com/api/v4/tasks/rrrrZ subTaskIdsFZBacklog)r,r customStatus followers responsiblesfollowdates)rrrrP attachmentZXMLHttpRequest)rz X-File-NamerzX-Requested-Withz /attachments)rrrZcustomStatusIdZ followerIdsZresponsibleIds)r,rrrrrZ superTasksrr userEmailZsendCopyrz%SPARC Reported Error/Issue SubmissionrZfeedbackzSPARC Feedback SubmissionZinterestz!SPARC Service Interest Submissiongeneralz$SPARC Question or Inquiry SubmissionzSPARC Research SubmissionzSPARC News SubmissionzSPARC Event SubmissionzSPARC Tool/Resource SubmissionzSPARC Story Submission)r,rtask_idrzMissing title or descriptionr)7rformrmrr<Z TURNSTILE_URLZNUXT_TURNSTILE_SECRET_KEYrrrrprgrrM WRIKE_TOKENZDRC_FEEDBACK_FOLDER_IDZCCB_HEAD_WRIKE_IDZDAT_CORE_TECH_LEAD_WRIKE_IDZMAP_CORE_TECH_LEAD_WRIKE_IDZK_CORE_TECH_LEAD_WRIKE_IDZSIM_CORE_TECH_LEAD_WRIKE_IDZMODERATOR_WRIKE_IDZDRC_WRIKE_CUSTOM_STATUS_IDZNEWS_AND_EVENTS_FOLDER_IDZCOMMS_LEAD_1_WRIKE_IDZCOMMS_LEAD_2_WRIKE_IDZCOMMS_LEAD_3_WRIKE_IDZCOMMS_WRIKE_CUSTOM_STATUS_IDZNEWS_TEMPLATE_TASK_IDZEVENT_TEMPLATE_TASK_IDZTOOLS_AND_RESOURCES_FOLDER_IDZ$TOOLS_AND_RESOURCES_TEMPLATE_TASK_IDZCOMMUNITY_SPOTLIGHT_FOLDER_IDZ$COMMUNITY_SPOTLIGHT_TEMPLATE_TASK_IDextendZ SUE_WRIKE_IDZ JYL_WRIKE_IDrrrfilenamerprintrr7rr5r9r6r8rrrrrr)#rZ captchaReqZ captchaResprr,rZnewTaskDescriptionhedrrrrZtaskTypeZtemplateTaskIdZtemplateSubTaskIdsZ templateUrlZ templateResprrNrZ new_task_idrZ file_data file_namerrZattachment_urlriZ subTaskIdZsubTaskTemplateUrlZsubTaskTemplateRespZ subTaskDatar6bodyrrUrUrVcreate_wrike_taskTs@$                 ,         rz/mailchimp_subscribec Cst}|rd|vrd|vrd|vr|d}|d}|d}tdtj}d|}|d||dd}z6tj|||d }|jd kr|WSd |WSWn0t y}zt d |WYd}~n d}~00d didfSt ddddS)N email_address first_name last_nameAnyUser;https://us2.api.mailchimp.com/3.0/lists/c81a347bd8/members/Z subscribed)ZFNAMEZLNAME)rrZ merge_fieldsrrauthrz(Failed to subscribe user with response: z!Could not subscribe to newsletterrgzAn error occured while trying to unsubscribe to the newsletterrrMissing email_addressrr)rbrrrrrNrrUrUrVunsubscribe_to_mailchimp0s(    " rz&/mailchimp_member_info/c Cs|rtdtj}d|}z4tj||d}|jdkr<|WSd|WSWn6ty}zt d||WYd}~n d}~00ddid fSt d d d dS) Nrr)rrrz)Failed to get member info with response: zFailed to get member info for rgz(Could not get member info from MailChimprrrr) r#r<rrmrrrrrrprgr)rrrrNrrUrUrVget_mailchimp_member_infoLs   ( rz/get-organ-curies/c Cstjd}t|}i}z,tjtjdtj|d}t | }Wn>t y}z&t d|ddddfWYd}~Sd}~00t|S) Nr7rrzFailed getting Uberon IDsrG BaseExceptionrIr)rrrr-rmrr<rrrCrrrprgr)r7Z requestBodyr<rrrUrUrVget_available_uberonidsds"  rz/get-related-terms/c Cstjjdddtjjdddtjjdddtjd}i}z*tjtjd ||d }t|}WnDt y}z,t d ||d d ddfWYd}~Sd}~00t |S)N directionZOUTGOING)rgrelationshipTypez BFO:0000050entailr)rrrr>z/graph/neighbors/)rz*Failed getting related terms with payload rGrrIr) rrrr<rrmZSCI_CRUNCH_SCIGRAPH_HOSTrErrrprgr)r$r@r<rrrUrUrVget_related_terms|s(rz /simulation_ui_file/cCsttt|}t|j}zd|dd}|d}|dddd}tdd|d |}td d|}ttt ||WSt yt d d d Yn0dS)Nr:rZs3urizabi-simulation-fileZdatasetrz s3://[^/]*/rPzfiles/z s3://|/.*rfz$no simulation UI file could be foundr) r@r"r+rrrresubrrrrr)r/r:Z results_jsonitemrrrZs3_bucket_namerUrUrVsimulation_ui_files   rz /pmr_filecCst}|rpd|vrpz@ttjd|d}|jdkrHt|j WS| WSWq|t dddYq|0n t ddddS)Nrrrrz invalid pathrz missing path) rrrmrr<ZPMR_HOSTrrrrrr)rrNrUrUrVpmr_files  rz/start_simulationcCsJt}|r:d|vr:d|dvr:d|dvr:tt|StddddS)Nsolverrrrz)Missing solver name and/or solver versionr)rrrrdo_start_simulationrrrUrUrVrJs$rJz/check_simulationcCsRt}|rBd|vrBd|vrBd|dvrBd|dvrBtt|StddddS)NZjob_idrrrrz1Missing solver name, solver version and/or job idr)rrrrdo_check_simulationrrrUrUrVrKs,rKz/pmr_latest_exposurecCst}|rd|vrzptj|dddid}|jdkrxz$|dddd dd }Wnd }Yn0t|d WS|WSWqtd ddYq0n td dddS)NZ workspace_urlAcceptz$application/vnd.physiome.pmr2.json.1r~r collectionitemsrZlinksZhrefrPrrzInvalid workspace URLrzMissing workspace URL)rrrmrrrrr)rrNrrUrUrVpmr_latest_exposures$   $ rz/onto_term_lookupc Cstjd}ddi}dtji}t|}zbtjtjd|||d}|}|dd}|dd}|d kr||d }|d } nd d i} | WSt y} zt d| WYd} ~ n d} ~ 00t dS)NrErrr>r?)rrrrmtotalrr_sourcelabelz not foundz.An error occured while fetching from SciCrunchr) rrrr<rr.rmZSCI_CRUNCH_INTERLEX_HOSTrrrrprgr) rErrr$rr:rmrr<rbrrUrUrVfind_by_onto_terms&    "rz/search-readme/c Csxd|}ddtji}ztj||d}|WStjjyr}z(t|t |dddfWYd}~Sd}~00dS)Nz2https://dash.readme.com/api/v1/docs/search?search=rzBasic rz9Readme is not currently reachable, please try again laterrFr) r<ZREADME_API_KEYrmrrrnrorprgrh)r$rrrrsrUrUrV search_readmes  rz/metricscCstSrQ)rrUrUrUrVmetricssrz/event_updatedcCsrtjd}|tjkr*tdd|dnDt}|rbz t|WStdd|dYqn0n tddddS)NZevent_updated_secret_keyr zInvalid secret key: rrzInvalid event data: zMissing event data)rrrr<ZCTF_CDA_ACCESS_TOKENrrr)Z secret_keyrrUrUrV event_updateds   rz/all_dataset_idscCs"t}dd|D}d}||S)NcSsg|] }t|qSrU)rh)ruelementrUrUrVry0rz#all_dataset_ids..z, )r join)listZ string_list delimiterrUrUrVall_dataset_ids-sr)rAr)F)atexitrZapp.metrics.pennsieverZapp.metrics.contentfulrrrZ!scripts.update_contentful_entriesrrZapp.metrics.algoliarr r r Zapp.metrics.gar r Zscripts.monthly_statsrZ"scripts.update_featured_dataset_idrrZapp.osparc.servicesrrZboto3rrprrm urllib.parserZ!apscheduler.schedulers.backgroundrZapscheduler.triggers.combiningrZapscheduler.triggers.daterZapscheduler.triggers.intervalrZbotocore.exceptionsrrrZflaskrrrrZ flask_corsrZflask_marshmallowrZ pennsiever Zpennsieve.baser!rqZPILr" requests.authr#Z flask_cachingr$Zapp.scicrunch_requestsr%r&r'r(r)r*r+r,r-r.r/r0r1r2r3Zscripts.email_senderr4r5r6r7r8r9 threadingr:Z xml.etreer; app.configr<Z app.dbtabler=r>r?Zapp.scicrunch_process_resultsr@rArBrCrDrErFZapp.serializerrGZ app.utilitiesrHrIZapp.osparc.osparcrJrrKrZapp.biolucida_process_resultsrrLr basicConfigrarZ DEPLOY_ENVrMcacheZmarrlclientZSPARC_PORTAL_AWS_KEYZSPARC_PORTAL_AWS_SECRETrNrZ DATABASE_URLrAttributeErrorrrobjectrOZ errorhandlerrjZbefore_first_requestrtrrrrrrrrZmonthly_stats_email_schedulermsZmonthly_stats_required_checkZadd_jobrTrrrrrrrrrZfeatured_dataset_id_triggerrregisterZrouterrrZDEFAULT_S3_BUCKET_NAMErrrrrrrrrrr!r%r'r)r+r-r1r3r4r8r=r"rDrKrMrOrZrrcrdrerjrlrorqcachedrtr|r}rrrrrrrrrrrrrrrrrrrrrrrrrrrUrUrUrVs                D    $                      >  '           ,   0                     8 #