o
    S=h                 
   @   sN  d dl Z d dlmZ d dlmZmZmZ d dlmZm	Z	 d dl
mZmZmZmZmZ d dlmZmZmZmZmZmZ d dlmZmZ d dlmZmZ d d	lmZ d dlZd dl Z d dl!Z!d dl"Z"d dl#Z#d dl$Z$d dl%Z%d dl&Z&d d
l'm(Z( d dl)Z)d dl*Z*d dl+Z+d dl,Z,d dl-Z-d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7 d dl8m9Z9 d dl:m;Z; d dl<m<Z<m=Z= d dl>m?Z?m@Z@mAZAmBZB d dlCmDZD d dlEmFZF d dlGmHZH d dlImJZJ d dlKmLZM d dlNmOZO d dlPmQZQ d dlRmSZS d dlTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZd d dlemfZfmgZgmhZhmiZimjZjmkZk d dllmmZm d dlnmoZo d d lpmqZq d d!lrmsZsmtZtmuZumvZvmwZw d d"lxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZ d d#lmZ d d$lmZmZmZ d d%lmZ d d&lmZ d d'lmyZmZ d dl-Z-e*  e?eZeqj Zejee*ee*j e;d(d)Zeqjejd*< eSed+d,d-d.ZeDe eFeZef Zdae!jd/eqjeqjd0d1Zem ZeqjZered2red2d3d4ZzeseZW n ey   dZY nw zeteZW n ey   dZY nw zeueZW n ey   dZY nw zeveZW n ey+   dZY nw zeweZW n ey>   dZY nw G d5d6 d6eZed7d8d9 Zejd:d; Zejd<d= Ze1 Ze1 Ze1 Ze1 Ze1 Ze1 ZeeZeqjd>kredu sed?d@kre  ejse*dA e  ejse*dB e  ere1 Ze   ejejdCdDdDdEdEd d dF eqjdGkreqjd>krejse*dH e  ejedCdIdJdK i ZejdLdM Zi Ze Ze Ze ZejdNdO Ze ZejdPdQ ZejedRd4dS ejedRd4dS ejedRdTdU e3e5 e7d4dVgZedWdX eΡ eqjd>kr=ejedCdYd d dZ d[d\ Ze eϡ ed]d^d_ Zejd`dagdbdcdd Zdedf ZedgdheqjfdidjZedkdldm ZedneqjfdodpZedqeqjfdrdsZdtdu Zedvdwdx Zdydz Zed{eqjfd|d}Zed~dd Zededdd Zeddd Zeddd Zededdd Zededdd Zededdd Zeddd Zeddd Zeddd Zeddd Zdd Zdd Zdd Zeddd ZedeqjfddZeddd Zdd Zejdddd ddeddwddZejdddideddd ZedddÄ Zddń ZddǄ ZddɄ Zedʡdd̄ Zed͡ddτ ZedСdd҄ ZedӡddՄ Zed֡dd؄ Zejddgdbdd܄ Zejddgdbdd߄ Zejddgdbejd,ddd Zejddgdbdd Zdd Z ejddgdbdd Zejddgdbdd Zejddgdbdd Zejddgdbdd Zejddgdbdd ZejddgdbdxddZejddgdbdd Zejddgdbd d Zejѐddgdbdd Z	ejѐddgdbdd Z
dd	 Zd
d Zdd Zejѐddagdbdd Zejѐddagdbdd Zejѐddagdbdd Zejѐddagdbdd Zejѐddagdbdd Zejѐddagdbdd Zd d! Zdyd"d#Zejѐd$dagdbd%d& Zd'd( Zdzd)d*Zd+d, Zd-d. Zejѐd/dagdbd0d1 Zejѐd2dagdbd3d4 Zejѐd5dgdbd6d7 Zejѐd8dagdbd9d: Zd;d< Zd=d> Z ejѐd?dagdbd@dA Z!eѐdBdCdD Z"eѐdEdFdG Z#eѐdHdIdJ Z$ejѐdKdagdbdLdM Z%ejѐdNdagdbdOdP ZejѐdQdagdbdRdS ZejѐdTdagdbdUdV Z&eѐdWdXdY Z'ejѐdZdgdbd[d\ Z(ejѐd]dgdbd^d_ Z)ejѐd`dgdbdadb Z*ejѐdcdgdbddde Z+ejѐdfdagdbdgdh Z,ejdidejѐdjdgdbdkdl Z-ejdidejѐdmdgdbdndo Z.eѐdpejdqddrds Z/ejѐdtdagdbdudv Z0dS ({      N)get_download_count)init_cf_cda_clientget_funded_projects_countget_featured_datasets)update_all_events_sort_orderupdate_event_sort_order)get_dataset_countinit_algolia_clientget_all_dataset_idsget_all_dataset_uuidsget_associated_datasets)init_ga_reportingget_ga_1year_sessionsinit_gspread_clientappend_contactupload_fileinit_drive_client)set_featured_dataset_id#get_featured_dataset_id_table_state)update_protocol_metrics get_protocol_metrics_table_state)OSparcServices)ApiException)urlparse)BackgroundScheduler)	OrTrigger)DateTrigger)IntervalTrigger)ClientError)ThreadPoolExecutor)datetime	timedelta)Flaskabortjsonifyrequest)CORS)Marshmallow)	Pennsieve)
new_client)UnauthorizedException)Image)HTTPBasicAuth)Cache)create_doi_querycreate_filter_requestcreate_facet_querycreate_doi_aggregatecreate_title_querycreate_identifier_query!create_pennsieve_identifier_querycreate_field_querycreate_request_body_for_curiescreate_onto_term_querycreate_multiple_doi_query create_multiple_discoverId_querycreate_anatomy_queryget_body_scaffold_dataset_idcreate_multiple_mimetype_querycreate_citations_query)EmailSenderfeedback_emailissue_reporting_email#creation_request_confirmation_email-anbc_form_creation_request_confirmation_email2service_form_submission_request_confirmation_email)Lock)ElementTree)Config)AnnotationTableMapTableScaffoldTableFeaturedDatasetIdSelectorTableProtocolMetricsTable)process_resultsprocess_get_first_scaffold_inforeform_aggregation_resultsreform_curies_resultsreform_dataset_resultsreform_related_termsreform_anatomy_results)ContactRequestSchema)img_to_base64_strget_path_from_mangled_listget_extension)start_simulation)check_simulation)rL   process_result   )Zmax_workersZENVZSimpleCache,  )Z
CACHE_TYPEZCACHE_DEFAULT_TIMEOUT)configs3z	us-east-1)Zaws_access_key_idZaws_secret_access_keyZregion_namezpostgres://zpostgresql://   c                   @   s`   e Zd ZdZe ed ZdZe	dd Z
dd Ze	dd	 Zd
d Ze	dd Zdd ZdS )	Biolucida i?B Fc                 C   
   | t _d S N)r_   _tokenvalue rf   7/home/cmiss/Jenkins/workspace/SPARC-API-DEV/app/main.py	set_token      
zBiolucida.set_tokenc                 C      | j S rb   )rc   selfrf   rf   rg   token      zBiolucida.tokenc                 C   ra   rb   )r_   _expiry_daterd   rf   rf   rg   set_expiry_date   ri   zBiolucida.set_expiry_datec                 C   rj   rb   )ro   rk   rf   rf   rg   expiry_date   rn   zBiolucida.expiry_datec                 C   ra   rb   )r_   _pending_authenticationrd   rf   rf   rg   set_pending_authentication   ri   z$Biolucida.set_pending_authenticationc                 C   rj   rb   )rr   rk   rf   rf   rg   pending_authentication   rn   z Biolucida.pending_authenticationN)__name__
__module____qualname__rc   r    nowr!   ro   rr   staticmethodrh   rm   rp   rq   rs   rt   rf   rf   rf   rg   r_   {   s    


r_     c                 C   s   t t| ddfS )N)errorrz   )r$   str)erf   rf   rg   resource_not_found   s   r~   c               
   C   s   zt tjtjdtjdaW d S  tjjy- }  zt	
d t	
|  W Y d } ~ d S d } ~ w tyI }  zt	
d t	
|  W Y d } ~ d S d } ~ w tye }  zt	
d t	
|  W Y d } ~ d S d } ~ ww )NF)Z	api_token
api_secretZenv_overridehostz#Unable to connect to Pennsieve hostz&Unable to authorise with Pennsieve ApizUnknown Error)r(   rF   PENNSIEVE_API_TOKENPENNSIEVE_API_SECRETPENNSIEVE_API_HOSTpsrequests
exceptions	HTTPErrorloggingr{   PSUnauthorizedException	Exceptionerrrf   rf   rg   connect_to_pennsieve   s(   


r   c               
   C   sT   zt tjtjtjd daW d S  ty) }  ztd|   W Y d } ~ d S d } ~ ww )N)api_keyr   Zapi_hostZ	api2_hostz'Error connecting to pennsieve 2 agent: )	r)   rF   r   r   r   ps2r   r   r{   r   rf   rf   rg   connect_to_pennsieve2   s   r   ZFALSEtotal_protocol_viewsz3Starting scheduler for protocol metrics acquisitionz6Starting scheduler for featured dataset id acquisitionZcron*2)yearmonthdayhourminutesecondZdevelopmentz8Starting scheduler for updating contentful event entries   z
US/Eastern)r   timezonec               
   C   s   t d z&tjtj dd} | jr*d| jddv r*|  }t	|d }|t
d< W n tyC } zt d	| W Y d }~nd }~ww tjsRt d
 t  d S d S )NzGetting oSPARC viewersz/viewersurlapplication/jsonzcontent-typer`   datafile_viewersz!Could not retreive oSPARC viewersz1Starting scheduler for oSPARC viewers acquisition)r   infor   getrF   OSPARC_API_HOSTokheadersjsonbuild_filetypes_tableosparc_datar   r{   viewers_schedulerrunningstart)reqZviewerstabler}   rf   rf   rg   get_osparc_file_viewers   s    

r   c                  C   st   t d trtt} | td< trtt}|td< tr#tt}|td< t	 }|td< t
js8t d t
  d S d S )NzGathering metrics dataZ1year_sessions_countZdataset_countZfunded_projects_countZ1year_download_countz*Starting scheduler for metrics acquisition)r   r   google_analyticsr   usage_metricsalgoliar   
contentfulr   r   metrics_schedulerr   r   )Zga_responseZalgolia_responseZcf_responseZps_responserf   rf   rg   get_metrics   s    

r   c               
   C   sn   t d ztjtj dd} |  }t|d  W d S  t	y6 } zt 
d| W Y d }~d S d }~ww )NzFetching oSPARC servicesz	/servicesr   r   z%Request to get oSPARC services failed)r   r   r   r   rF   r   r   osparc_servicesZset_servicesr   r{   )r   Zservices_respr}   rf   rf   rg   get_services  s   
r   interval)functriggerdays   )r   r   hours)r   c                   C      t tS rb   )r   featuredDatasetIdSelectorTablerf   rf   rf   rg   <lambda>'  s    r   sat)Zday_of_weekr   r   c                   C   s   t d tjrt  t d tjrt  t d tjr$t  t d tjr0t  t d tjr<t  t d t	jrJt	  d S d S )Nz1Stopping scheduler for oSPARC viewers acquisitionz*Stopping scheduler for metrics acquisitionz2Stopping scheduler for updating contentful entriesz3Stopping scheduler for updating featured dataset idz0Stopping scheduler for updating protocol metricsz&Stopping scheduler for oSPARC services)
r   r   r   r   shutdownr   )update_contentful_event_entries_schedulerfeatured_dataset_id_schedulerprotocol_metrics_schedulerservices_schedulerrf   rf   rf   rg   shutdown_schedulers-  s&   





r   z/healthc                   C   s   t ddiS )NstatusZhealthy)r   dumpsrf   rf   rf   rg   healthE  s   r   z/contactPOST)methodsc               	   C   sh   t tj} t | }|d }|d }|d }t||| tt	j
|dtd|i t ddiS )NnameemailmessagezFeedback submissionr   sent)r   loadsr%   r   rS   loademail_senderZ
send_emailmailersend_emailrF   
SES_SENDERr?   
substituter   )r   Zcontact_requestr   r   r   rf   rf   rg   contactJ  s   r   c                 C   s   t jd| |d|d|d}|S )N
get_object	requester)BucketKeyRequestPayerZResponseContentType)ZParamsZ	ExpiresIn)r]   Zgenerate_presigned_url)s3BucketNamekeycontent_type
expirationresponserf   rf   rg   create_s3_presigned_urlY  s   r   z	/downloadi  c                 C   s6   t jd}t jd|}t jdd}t|||| S )Nr   r   ZcontentTypezapplication/octet-stream)r%   argsr   r   )r   bucket_namer   r   r   rf   rf   rg   create_presigned_urld  s   r   z/thumbnail/neurolucidac               
   C   s  t j} d| vsd| vsd| vrtdddS tj d}z)tj|| dd	}|  |jd
kr=|j	dddkr=t
|jW S tdd W d S  tjjyU   tddd Y S  tjjyd   tdd Y S  tjjy } ztddt| W  Y d }~S d }~ww )NversionZ	datasetIdpath  Query arguments are not valid.descriptionz
/thumbnail   )paramstimeout   Content-Typeunknownz	image/pngzFailed to retrieve thumbnail.z0Unable to make a connection to NEUROLUCIDA_HOST.i  z&Request to NEUROLUCIDA_HOST timed out.  z)Error while requesting NEUROLUCIDA_HOST: )r%   r   r#   rF   ZNEUROLUCIDA_HOSTr   r   raise_for_statusstatus_coder   base64	b64encodecontentr   ConnectionErrorZTimeoutRequestExceptionr|   )
query_argsr   r   r}   rf   rf   rg   thumbnail_from_neurolucida_filem  s&   
 r   z/thumbnail/segmentationc              
   C   s  t j}d|vrtdddS |d| }|d }d}d}d}d}d	}|}	|r'|sztj||d
| d|	 dd}
W n5 tym } z)|jd d dkrYtdd| ddW  Y d}~S tdd| ddW  Y d}~S d}~ww |
d  	d}d|v }d|v }|r|s|	|7 }	n||7 }|	|7 }	t
||k rtdd| ddS |r'|r'|du rtdd| ddS ||dd }|d|d d }t|}|j}d}|D ]}||jdd 7 }qt|}tdt|d t|d f|}t|}|S )z
    Extract a thumbnail from a mbf xml file.
    First phase is to find the thumbnail element in the xml document.
    Second phase is to convert the xml to a base64 png.
    r   r   r   r   r   NFr   i  zbytes=-r   )r   r   ZRanger   ErrorCode	NoSuchKeyrz   Could not find file: ''Unknown error for file: 'BodyzUTF-8z<thumbnail z</thumbnail>z#Could not find thumbnail in file: 'r`   r   ZRGBrowscols)r%   r   r#   r   r]   r   r   r   readdecodelenfindrE   
fromstringattribtextbytesfromhexr+   	frombytesintrT   )r   r   r   r   resourceZstart_tag_foundZend_tag_foundZ
start_byteoffsetZend_byter   exZstart_thumbnail_elementZthumbnail_xmlxmlZ	size_infoZim_datachildZbyte_im_dataZimZbase64_formrf   rf   rg   extract_thumbnail_from_xml_file  s`   
  


"r  z/exists/<path:path>c                 C   sf   t j}|d|}z
tj|| dd}W n ty    ddi Y S w |dd}|dkr/ddiS ddiS )	Nr   r   r   r   r   existsfalseContentLengthr   true)r%   r   r   r]   head_objectr   )r   r   r   r   head_responsecontent_lengthrf   rf   rg   
url_exists  s   
r  c                 C   s0   |  dd} | d | d } t| }| S )Nzhttps://api.pennsieve.io/z"https://api.pennsieve.io/discover//)replacerfindr   r   r   )urirrf   rf   rg   fetch_discover_file_information  s   
r%  z/s3-resource/discover_pathc               
   C   s   t jd} zt| }d|v r |d dkr |d d }|d W S W n ty9 } ztd| W Y d }~nd }~ww tdd	|  d
S )Nr#  Z
totalCountr^   filesr   r   zFailed to retrieve uri {uri}rz   zFailed to retrieve uri r   )r%   r   r   r%  r   r   r{   r#   )r#  json_response	file_infor  rf   rf   rg   get_discover_path  s   
r)  c              
   C   s   z&t j|| dd}|dtj}|r!|tjk s$tdd| dW S W dS W dS  tjjyk } z7|j	d d d	krAdW  Y d }~S |j	d d dkrRdW  Y d }~S t|j	d d |j	d d W  Y d }~S d }~ww )Nr   r  r  i  zFile too big to download: r   r   r   Z404rz   .Provided path was not found on the s3 resourceZ403  EThere is a permission issue when accessing the file at specified pathMessage)r   OK)rz   r*  )r+  r,  )
r]   r  r   rF   ZDIRECT_DOWNLOAD_LIMITr#   botocorer   r   r   )r   r   r  r  r   rf   rf   rg   s3_header_check  s*   *r0  z/s3-resource/<path:path>c           
      C   s   t j}|d|}| }t| |}|d dks|d dkrQt| }||kr*tddd t||}|d dkr8|}n|d dkrEtddd n|d dkrQtddd tj||d	d
}t jd}|d  }	|d urnt	
|	S |	S )Nr   r   rz   r+  r*  r   r   r,  r   r  ZencodeBase64r  )r%   r   r   r0  rU   r#   r]   r   r  r   r   )
r   r   r   r   Zs3_pathr   Zs3_path_modifiedZ	response2encode_base64r  rf   rf   rg   direct_download_url
  s2   


r2  z /scicrunch-dataset/<doi1>/<doi2>c              
   C   s   |  ddd | }t|}ztjtj dtj |d}| W S  tjj	yA } zt
| td|iW  Y d }~S d }~ww )NDOI:r`   r   /_search?api_key=r   r{   )r!  r.   r   postrF   SCI_CRUNCH_HOSTKNOWLEDGEBASE_KEYr   r   r   r   r{   r   )Zdoi1Zdoi2doir   r   r   rf   rf   rg   sci_doi0  s   

r:  z/pubmed/<id_>z/pubmed/<id_>/c              
   C   s\   zt d|  d}|jW S  t jjy- } zt| td|iW  Y d }~S d }~ww )Nz https://pubmed.ncbi.nlm.nih.gov/r   r{   )	r   r   r  r   r   r   r{   r   r   )id_r   r   rf   rf   rg   pubmed@  s   
r<  z/scicrunch-query-string/c               
   C   s   t jd} t jd}t jd}t jd}t| |||}ztjtj dtj	 |d}t
| W S  tjjyT } zt| td|iW  Y d }~S d }~ww )Nfieldcuriesizefromr4  r5  r{   )r%   r   getlistr   r5   r   r6  rF   r7  r8  rL   r   r   r   r   r{   r   )fieldsr>  r?  Zfrom_r   r   r   rf   rf   rg   	sci_organL  s    
rC  z/dataset_info/using_doic                  C   s<   t jd} t jd}t| }|d u rtt|S t|S )Nr9  Zraw_response)r%   r   r   r.   rP   dataset_search)r9  rawqueryrf   rf   rg   get_dataset_info_doi_  s   rG  z!/dataset_info/using_multiple_doisz"/dataset_info/using_multiple_dois/c                  C       t jd} t| }tt|S )Ndois)r%   r   rA  r8   rL   rD  )rI  rF  rf   rf   rg   get_dataset_info_doisk     rJ  z./multiple_dataset_info/using_multiple_mimetypez//multiple_dataset_info/using_multiple_mimetype/c                  C   rH  )Nq)r%   r   rA  r<   rL   rD  )rL  rF  rf   rf   rg   get_file_info_from_mimetypet  s   rM  z(/dataset_info/using_multiple_discoverIdsz)/dataset_info/using_multiple_discoverIds/c                  C   rH  )NdiscoverIds)r%   r   rA  r9   rL   rD  )rN  rF  rf   rf   rg   get_dataset_info_discoverIds~  rK  rO  z/dataset_info/using_titlec                  C   rH  )Ntitle)r%   r   r   r2   rP   rD  )rP  rF  rf   rf   rg   get_dataset_info_title     rQ  z%/dataset_info/using_object_identifierc                  C   rH  N
identifier)r%   r   r   r3   rP   rD  rT  rF  rf   rf   rg   "get_dataset_info_object_identifier  rR  rV  z/dataset_info/anatomyc                  C   s6   t jdd} | dkrtdddS t| }tt|S )NrT  r   rz   z Identifier for API call not set.r   )r%   r   r   r#   r:   rR   rD  rU  rf   rf   rg   get_dataset_info_anatomy  s
   rW  z(/dataset_info/using_pennsieve_identifierc                  C   rH  rS  )r%   r   r   r4   rP   rD  rU  rf   rf   rg   %get_dataset_info_pennsieve_identifier  rR  rX  c              
   C   s  g }|D ]}|r| d|ks|r| di  dd|kr| di }| di  d}	| di }
|
 d}|
 d}| d	i  d}|rX|| |d
 |d d ||d |	d ur|	D ]7}|dd}t| ||d ||}t|dkr{|| q^|d u s|d u r| |d
 |d d |d}|| q^|d ur|d urtt|D ]%}d|  krt|k rn q|| dd}|td |d || | qq|S )NrT  datasetr   r`   dataciteZisDerivedFromZderived_from_datasetr#  Zassociated_flatmapr   )
discoverIdr   r   r   flatmapUUIDz'derivative/sub-f006/derivative/sub-f006zderivative/sub-f006r   )r[  r   r   r   zhttps://doi.org/)r   appendr!  +get_is_derived_from_with_identifier_or_pathr  extendrangeget_original_source)r[  r   objectsrT  matchingPathdatasetCacheZsource_listitemrZ  ZisDerivedFromPathsZderivedfromdatasetZderivedfromdatasetdoiZderivedfromdatasetpathr\  r   Zsource_objectsr   ir9  rf   rf   rg   r^    sT   


	

r^  c                 C   s   |  di  dg }t|dkrR|d  di  d}|d  di  di  d}|d  di  di  di  d}|d urR|d urR|d urRt||||||S g S )	Nhitsr^   r   _sourcerb  	pennsieverT  r   )r   r  r^  )dataset_inforT  rc  rd  rg  rb  r[  r   rf   rf   rg   get_original_source_in_dataset  s   &rk  c           	      C   s   d }| }| d urt | }n!|d ur.|d ur#|}|dd}t|g}n|d ur.|}t|g}||d }|d u r@t|}|||< t|| ||S )Nr3  r`   )r3   r!  r8   r9   r   rD  rk  )	rT  r9  r[  r   rd  rF  idZnewDOIrj  rf   rf   rg   ra    s"   

ra  z/file_info/get_original_sourcec                  C   sH   t jd} t jd}t jd}t jd}i }dt||| ||iS )Nr[  r9  rT  r   result)r%   r   r   ra  )r[  r9  rT  r   rd  rf   rf   rg   get_file_info_original_source  s   rn  z/segmentation_info/c              
   C   sL  t j}d|vrtdddS |d| }|d}zt||}|d dkr*tddd tj||d	d
}W n5 tyh } z)|jd d dkrTtdd| ddW  Y d }~S tdd| ddW  Y d }~S d }~ww |d 	 }t
|}|d}i }	|d ur|j|	d< n	ddddd|	d< |d}
|
d ur|
j|	d< |	S ddi|	d< |	S )Ndataset_pathr   z,Query arguments must include 'dataset_path'.r   r   r   rz   r*  r   r  r   r   r   r   r  r  r  z./{*}sparcdata/{*}subjectsubjectr`   )ZageZsexspeciesZ	subjectidz./{*}sparcdata/{*}atlasZatlasZorgan)r%   r   r#   r   r0  r]   r   r   r   r  rE   r
  r	  r  )r   r   r   ro  r   r  r  r  Zsubject_elementr   Zatlas_elementrf   rf   rg   get_segmentation_info_from_file  sB   


  



rr  z/current_doi_listc                  C   sB   t  } tt| }g }|d d D ]}||d d  qd|iS )Nr9  bucketsr   r>  results)r1   rN   rD  r]  )rF  rt  Zdoi_resultsrm  rf   rf   rg   get_all_doi0  s   ru  c              
   C   sp   z| }dt ji}tjt j d||d}| W S  tjjy7 } zt	| t
d|iW  Y d }~S d }~ww )Nr   /_search)r   r   r{   )rF   r8  r   r6  r7  r   r   r   r   r{   r$   )rF  payloadr   r   r   rf   rf   rg   rD  ;  s   

rD  z/search/r`   
   )rF  limitr   )defaultsz/search/<query>c                 C   s   zFt jdd urt jd}t jdd urt jd} t jdd ur+t jd}ttj d|  d| d| dtj 	}t| W S  tj	j
yf } zt| td|iW  Y d }~S d }~ww )	Nry  rF  r   z/_search?q=z&size=z&from=z	&api_key=r{   )r%   r   r   r   rF   r7  r8  rL   r   r   r   r   r{   r   )rF  ry  r   r   r   rf   rf   rg   	kb_searchM  s   *
r{  z/filter-search/rF  z/filter-search/<query>/c           	   
   C   s   t jd}t jd}t jd}t jd}t| ||||}ztjtj dtj	 |d}t
| }W |S  tjjy[ } zt| tt|ddd	fW  Y d }~S d }~w tjym   td
ddd	f Y S w )NtermZfacetr?  r   r4  r5  z<SciCrunch is not currently reachable, please try again laterr{   r   r   8Could not parse SciCrunch output, please try again laterJSONDecodeErrorr   r{   )r%   r   rA  r   r/   r   r6  rF   r7  r8  rL   r   r   r   r   r{   r$   r|   r  )	rF  termsZfacetsr?  r   r   r   rt  r   rf   rf   rg   filter_searcha  s.   
"r  z/get-facets/<type_>c           
      C   s   t | \}}g }||  D ]U}||d |   d d< ztjtj dtj |d}| }|| W q tjyG   t	dddd	f Y   S  t
ya } ztd
| | W Y d }~qd }~ww g }|D ]}	||	d |   d 7 }qft	|S )Naggregationsr  r=  r4  r5  r~  r  r  r   z$Could not search SciCrunch for path rs  )r0   r   r6  rF   r7  r8  r   r]  r  r$   r   r   r{   )
type_Ztype_mapr   rt  r   r   Zjson_resultr  r  rm  rf   rf   rg   
get_facets|  s2   r  c                 C   s*   d| v rt | d}|j| d< d S d S )NZreadmemarkdown)r   r   r  )respZmark_reqrf   rf   rg   inject_markdown  s   r  c           
      C   s@  |  d}|  d}|d u s|d u rd S t|}|j}ztj|d|dd}W n> tyf   tjdk}|r;t	
d ztj|d|dd}W n tyc } z|rWt	| W Y d }~Y d S d }~ww Y nw |d	  }zt|}	W n ty } zt	| W Y d }~d S d }~ww |	 d
|	 d|	 dd| d< d S )Nrl  r#  z{}/files/template.jsonr   r  ZTRUEzXRequired file template.json was not found under /files folder, trying under /packages...z{}/packages/template.jsonr  uuidr   r   )r  r   r   Zstudy)r   r   netlocr]   r   formatr   rF   SPARC_API_DEBUGGINGr   warningr{   r  r   r   
ValueError)
r  r;  r#  Z
parsed_uribucketr   	debuggingr}   templateZtemplate_jsonrf   rf   rg   inject_template_data  sX   






r  c                 C   sF   i }| D ]}|d   }|d= ||dsg ||< || | q|S )N	file_typeF)lowerr   r]  )Zosparc_viewersr   ZviewerZfiletyperf   rf   rg   r     s   r   z/sim/dataset/<id_>c              
   C   s   t jdkrGz tdtj| }|jr$| }t	| t
| t|W S W n ty@ } ztd|  | W Y d }~nd }~ww tdddS d S )NGETz{}/datasets/{}Could not fetch SIM dataset rz   Resource not foundr   r%   methodr   r   r  rF   DISCOVER_API_HOSTr   r   r  r  r$   r   r   r{   r#   )r;  r   	json_datar  rf   rf   rg   sim_dataset  s   

r  z&/sim/dataset/<id_>/versions/<version_>c              
   C   s   t jdkrKz!tdtj| |}|jr%| }t	| t
| t|W S W n tyD } ztd|  d| | W Y d }~nd }~ww tdddS d S )Nr  z{}/datasets/{}/versions/{}r  z	 version rz   r  r   r  )r;  Zversion_r   r  r  rf   rf   rg   sim_dataset_versions  s   

$r  z/get_osparc_datac                   C   r   rb   )r$   r   rf   rf   rf   rg   get_osparc_data     r  z/sim/servicec                  C   sT   t jdkr(t jd} t jjddtd}t jjddtd}t| ||}t|S d S )Nr  searchry  r   )defaulttypeskipr   )r%   r  r   r   r  r   Zsearch_servicesr$   )r  ry  r  rt  rf   rf   rg   osparc_search  s   
r  z	/sim/filec                  C   s    t jdkrtj} td| iS d S )Nr  r   )r%   r  r   Zfile_extensionsr$   )
extensionsrf   rf   rg   osparc_extensions  s   
r  z/project/<project_id>r  c                 C   s4   t | }t|d dkrt|d S tddd d S )Nrg  r   rz   r  r   )r   r  r$   r#   )Z
project_iddatasetsrf   rf   rg   datasets_by_project_id  s   r  z"/get_featured_datasets_identifiersc                   C   s
   dt  iS )Nidentifiers)r   rf   rf   rf   rg   !get_featured_datasets_identifiers  ri   r  z/get_featured_dataset)r   c               
   C   s   t td } | dkrd} z!tdtj|  }|d g kr+tdtjd }|W S  tyH } zt	
d|  | W Y d }~nd }~ww tddd	 d S )
Nfeatured_dataset_idr       z{}/datasets?ids={}r  zCould not get featured dataset rz   z,An error occured while fetching the resourcer   )r   r   r   r   r  rF   r  r   r   r   r{   r#   )r  r   r  rf   rf   rg   get_featured_dataset  s   r  z/reva/subject-idsc               
   C   s   z/t dtj } | d }g }|D ]}|d d dkr%||d d  qtd|dd	fW S  tyR } ztd
|  td
|ddfW  Y d }~S d }~ww )N
/packages/childrenr   packageType
Collectionr   success)r   idsr   z+Error while getting REVA subject id files: r   r     )	r   r   rF   ,REVA_3D_TRACING_PRIMARY_FOLDER_COLLECTION_IDr]  r$   r   r   r{   )primary_folderprimary_childrenZsubject_idsr  r}   rf   rf   rg   getRevaSubjectIds1  s   r  c              
      sX  zd dt dtj }|s#dtj }t| td|dW S |dg }tfdd	|D d }|d u rJd
 }t| td|dW S t d|d d  }|smd|d d  }t| td|dW S |dg }t fdd	|D d }|d u rd }t| td|dW S t d|d d  }|sd|d d  }t| td|dW S |dg }	tfdd	|	D d }
|
d u rd }t| td|dW S t d|
d d  }|sd|
d d  }t| td|dW S |dg W S  ty+ } zd| }t| td|dW  Y d }~S d }~ww )NZCoordinatesDataZInSitur  zPrimary folder not found: rz   r   r  c                 3   $    | ]}|d  d  kr|V  qdS r   r   Nrf   .0r  
subject_idrf   rg   	<genexpr>K     " z5getRevaTracingInSituFolderChildren.<locals>.<genexpr>z)Subject folder not found for subject id: r   rl  z,Subject folder could not be fetched for id: c                 3   r  r  rf   r  )coordinates_folder_namerf   rg   r  X  r  z.CoordinatesData folder not found for subject: z4CoordinatesData folder could not be fetched for id: c                 3   r  r  rf   r  )in_situ_folder_namerf   rg   r  e  r  z%InSitu folder not found for subject: z+InSitu folder could not be fetched for id: z2Exception thrown when getting Reva InSitu Folder: r  )	r   r   rF   r  r   r{   r#   nextr   )r  r  msgr  subject_childsubject_foldersubject_childrenZcoordinates_childZcoordinates_folderZcoordinates_childrenZin_situ_childZin_situ_folderr}   rf   )r  r  r  rg   "getRevaTracingInSituFolderChildren@  s`   











r  z-/reva/anatomical-landmarks-files/<subject_id>c              
      s  zd t | }t fdd|D d }|d u r1td  d|   td  d|  dd	fW S td
|d d  }|d }g }|D ]_}|d d }|d d }td
| }	|	d }
g }|
D ]6}|d d }td
| d}|d d d }td
| d| d }|t|d d t|d qc|t||d qEtd|ddfW S  t	y } ztd|  td|ddfW  Y d }~S d }~ww )NZAnatomicalLandmarksc                 3   r  r  rf   r  Z anatomical_landmarks_folder_namerf   rg   r    r  z2getRevaAnatomicalLandmarksFiles.<locals>.<genexpr>REVA tracing folder  not found for subject: ERROR folder not found for subject: r  rz   r  r   rl  r  r   /viewr   /files/r   )r   s3Url)r   r&  r  )r   foldersr   z4Error while getting REVA anatomical landmarks files z0Error while getting anatomical landmarks files: r  
r  r  r   r{   r$   r   r   r]  r|   r   )r  in_situ_childrenZanatomical_landmarks_childZanatomical_landmarks_folderZanatomical_landmarks_childrenZanatomical_landmarks_foldersZanatomical_landmark_childZlandmark_folder_nameZlandmark_folder_idZanatomical_landmark_folderZlandmark_childrenZlandmark_filesZlandmark_childZlandmark_file_package_idZlandmark_fileZlandmark_file_idZlandmark_file_presigned_urlr}   rf   r  rg   getRevaAnatomicalLandmarksFilesz  s:   "r  z /reva/tracing-files/<subject_id>c              
      s  zd t | }t fdd|D d }|d u r1td  d|   td  d|  dd	fW S td
|d d  }|d }g }|D ]R}td
|d d  }|d }|D ]=}	|	d d }
td
|
 d}|d d d }td
|
 d| d }|t|	d d t|d d t|d qYqEtd|ddfW S  t	y } ztd|  td|ddfW  Y d }~S d }~ww )NZ
VagusNervec                 3   r  r  rf   r  Zvagus_nerve_folder_namerf   rg   r    r  z&getRevaTracingFiles.<locals>.<genexpr>r  r  r  r  r  rz   r  r   rl  r  r  r   r  r   r   )r   regionr  r  r   r&  r   z'Error while getting REVA tracing files z#Error while getting tracing files: r  r  )r  r  Zvagus_nerve_childZvagus_nerve_folderZvagus_nerve_childrenZvagus_tracing_filesZvagus_region_childZvagus_region_folderZvagus_region_childrenZvagus_file_childfile_package_idZ
vagus_fileZvagus_file_idZvagus_file_presigned_urlr}   rf   r  rg   getRevaTracingFiles  s8   &r  z!/reva/micro-ct-files/<subject_id>c              
      s   d zt dtj }|d }tfdd|D d }|d u r8td  tdd d	d
fW S t d|d d  }|d }t fdd|D d }|d u rrtd  d  td  d d	d
fW S t d|d d  }|d }g }	|D ]S}
|
d d }t d| d}|d d d }t d| d| d }|
d d }|
d }|
d d }|d d d }|	t	|t	|t	|t	|t	|d qtd|	ddfW S  t
y } ztd|  td|d	dfW  Y d }~S d }~ww )Nz-MicroCTVisualizationr  r  c                 3   r  r  rf   r  r  rf   rg   r    r  z&getRevaMicroCtFiles.<locals>.<genexpr>z/REVA microCT folder not found with subject id: r  z*MicroCT folder not found with subject id: r  rz   r   rl  c                 3   r  r  rf   r  )"micro_ct_visualization_folder_namerf   rg   r    r  zREVA microCT r  r  r   r  r   r   Zstorager  ZfileType)r   r  r  r  r?  r  r  r   z'Error while getting REVA microCT files z#Error while getting microCT files: r  )r   r   rF   Z*REVA_MICRO_CT_PRIMARY_FOLDER_COLLECTION_IDr  r   r{   r$   r]  r|   r   )r  r  r  r  r  r  Zmicro_ct_childZmicro_ct_visualization_folderZmicro_ct_childrenZmicro_ct_filesZmicro_childr  Zmicro_child_fileZmicro_child_file_idZmicro_file_presigned_url	file_name	file_sizeZpackage_typer  r}   rf   )r  r  rg   getRevaMicroCtFiles  sF   
"r  z/get_owner_email/<int:owner_id>c                    sN   t jj}t jj|} fdd|D }|stddd d S td|d jiS )Nc                    s   g | ]	}|j  kr|qS rf   )Zint_id)r  xowner_idrf   rg   
<listcomp>  s    z#get_owner_email.<locals>.<listcomp>rz   zOwner not foundr   r   r   )r   Z_apiZ_organizationZorganizationsZget_membersr#   r$   r   )r  orgmembersresrf   r  rg   get_owner_email  s   r  z!/get_body_scaffold_info/<species>c                 C   s:   t | }|rt|}tt|}|r|S tdd|  dS )Nrz   zWhole body info not found for r   )r;   r4   rM   rD  r#   )rq  rl  rF  rm  rf   rf   rg   get_body_scaffold_info  s   r  z/thumbnail/<image_id>Fc              
   C   s   t  }zZt | st  W d    n1 sw   Y  tjd|  }d| i}tjd||d}t	
|j}|dkr[|s[t |d W d    n1 sQw   Y  t| d}|W S  tyx } ztd|  | W Y d }~nd }~ww td	d
S )Nz/thumbnail/{0}rm   r  r   s   eyJzdGF0dXMiOiJBZG1pbiB1c2VyIGF1dGhlbnRpY2F0aW9uIHJlcXVpcmVkIHRvIHZpZXcvZWRpdCB1c2VyIGluZm8uIFlvdSBtYXkgbmVlZCB0byBsb2cgb3V0IGFuZCBsb2cgYmFjayBpbiB0byByZXZlcmlmeSB5b3VyIGNyZWRlbnRpYWxzLiJ9r`   Tz Could not get the thumbnail for rz   z-An error occured while fetching the thumbnail)r_   biolucida_lockrm   authenticate_biolucidarF   BIOLUCIDA_ENDPOINTr  r   r%   r   r   r   rh   thumbnail_by_image_idr   r   r{   r#   )image_idZrecursive_callblr   r   r   encoded_contentr  rf   rf   rg   r    s2   

r  z/image/<image_id>c              
   C   sl   t jd|  }ztd|}t| W S  ty0 } zt	d|  | W Y d }~nd }~ww t
ddS )Nz/image/info/{0}r  zCould not get image info for rz   z/An error occured while getting the image's info)rF   r  r  r   r%   process_biolucida_resultr   r   r   r{   r#   )r  r   r   r  rf   rf   rg   image_info_by_image_id  s   
r  z/image_search/<dataset_id>c              
   C   sj   t jd|  }ztd|}| W S  ty. } ztd|  | W Y d }~nd }~ww ddidfS )Nz%/imagemap/search_dataset/discover/{0}r  z$Could not search images for dataset r{   z3An error occured while searching images for datasetrz   )	rF   r  r  r   r%   r   r   r   r{   )
dataset_idr   r   r  rf   rf   rg   image_search_by_dataset_id"  s   
r  z/image_xmp_info/<image_id>c                 C   sv   t jd|  }ztd|}W n tjjy!   tddd Y S w | }|d dkr2t	|d S tdd	|  dS )
Nz/image/xmpmetadata/{0}r  r   )Unable to make a connection to Biolucida.r   r   r  r   zXMP info not found for )
rF   r  r  r   r%   r   r   r#   r   process_biolucida_resultsr  r   rm  r   rf   rf   rg   image_xmp_info-  s   r  z/image_blv_link/<image_id>c                 C   sz   t jd|  }ztd|}W n tjjy!   tddd Y S w | }|d dkr4t	d|d iS tdd	|  dS )
Nz/image/blv_link/{0}r  r   r  r   r   r  linkzBLV link not found for )
rF   r  r  r   r%   r   r   r#   r   r$   r  rf   rf   rg   image_blv_link<  s   r  c                  C   sh   t  } tjd }tjtjdd}g }i }tjd||||d}|jtjj	kr2|
 }| |d  d S d S )Nz/authenticater`   )usernamepasswordrm   r   )r   r   r&  rm   )r_   rF   r  ZBIOLUCIDA_USERNAMEZBIOLUCIDA_PASSWORDr   r%   r   codesr   r   rh   )r  r   rw  r&  r   r   r   rf   rf   rg   r  K  s   
r  c                 C   s\   d}| r&t  }|rd|v r|d }| ||}td|iS tddd d S tddd d S )	NTstater  r   zState not specifiedr   rz   Database not available)r%   get_jsonZ	pushStater$   r#   )r   Zcommitr  r  r  rf   rf   rg   get_share_link\  s   r   c                 C   s`   | r(t  }|r d|v r |d }| |}|r td| |iS tddd d S tddd d S )Nr  r  r   z#Key missing or did not find a matchr   rz   r  )r%   r  Z	pullStater$   r#   )r   r  r  r  rf   rf   rg   get_saved_statej  s   
r  z/annotation/getshareidc                   C   r   rb   )r   annotationtablerf   rf   rf   rg   get_annotation_share_linkx  r  r  z/annotation/getstatec                   C   r   rb   )r  r  rf   rf   rf   rg   get_annotation_state~  r  r  z/map/getshareidc                   C   r   rb   )r   maptablerf   rf   rf   rg   get_map_share_link  r  r  z/map/getstatec                   C   r   rb   )r  r  rf   rf   rf   rg   get_map_state  r  r  z/scaffold/getshareidc                   C   r   rb   )r   scaffoldtablerf   rf   rf   rg   get_scaffold_share_link  r  r	  z/scaffold/getstatec                   C   r   rb   )r  r  rf   rf   rf   rg   get_scaffold_state  r  r
  c              
   C   s   z&t jtjtj| dd}| }d|vs|d s ddidfW S |ddW S  ty? } zt	d| W Y d }~d S d }~ww )	NZsecretr   r   r   r  r{   Failed Captcha Validation  F0Could not validate captcha, bypassing validation)
r   r6  rF   TURNSTILE_URLNUXT_TURNSTILE_SECRET_KEYr   r   r   r   r{   )rm   
captchaReqcaptchaRespr  rf   rf   rg   verify_recaptcha  s   r  c           	      C   s   dt j dt j d}dt j dd}| |d}|r||d< |r%||d	< tj|||d
}|jdkrB| }|d |d |d dS td|j	 )Nhttps://api.github.com/repos/r   z/issuestoken application/vnd.github+jsonAuthorizationAccept)rP  bodylabels	assigneesr   r      html_urlcomments_urlr   )r   r!  issue_api_urlzGitHub Issue creation failed: )
rF   SPARC_GITHUB_ORGSPARC_ISSUES_GITHUB_REPOSPARC_TECH_LEADS_GITHUB_TOKENr   r6  r   r   r   r  )	rP  r  r  r  r   r   r   r   Zresponse_jsonrf   rf   rg   create_github_issue  s&   

r&  z/create_issuec                
   C   sP  t j} t jd}tjd s|rt|stddidfS | dd}| d}| d	}|r0|s6tdd
d | dd }|dv ryzt	| ||gt
jd}|d }|d }|d }	W n' tyx }
 ztdt|
idfW  Y d }
~
S d }
~
ww tdd| idfS d}d}d}|rd}d}d}t j}|rSd|v rS|d }| }|j}t d}t j}| d| d| }dt
j dt
j d| }dt
j d d!d"}t|d#}d$| |d%}zQtj|||d&}|jd'v r.|  }|d( d) }d*| d+}dt
j d d,}d	|i}tj!|||d-}|jdkr)|d.7 }d}d/}n|d07 }n|d17 }d}d/}W n tyR }
 z|d17 }d}d/}W Y d }
~
nd }
~
ww |rd2}t"#d3|i}|d4krmd5}t$#d3|i}t%%|}zt&'t
j(||| |d67 }W n ty }
 z|d77 }d}d/}W Y d }
~
nd }
~
ww t|||	|d8|fS )9Ncaptcha_tokenTESTINGr{   Invalid reCAPTCHAr   r  bugrP  r  zMissing title or bodyr   r   r`   )r*  feedbacktest)r  r  r   r!  r"  r  Unsupported task type: zSubmission could not be createdz!Submission created successfully. r  r  
attachmentz%Y%m%d-%H%M%S_r  r   z/contents/attachments/r  r  z
2022-11-28)r  r  zX-GitHub-Api-Versionutf-8z	Add file )r   r   r   r   )r   r  r   Zdownload_urlz![Issue Attachment]()r  r  zFile attachment unsuccessful. r  zFile attachment successful. zFile upload unsuccessful. zSPARC Reported Issue Submissionr   r+  z"SPARC Reported Feedback Submissionz,Confirmation email sent to user successful. .Confirmation email sent to user unsuccessful. )r   r   r"  r   ))r%   formr   appr\   r  r$   r#   stripr&  rF   ZGITHUB_ISSUE_ASSIGNEESr   r|   r&  r  filenamer    rx   strftimer  uuid4hexr#  r$  r%  r   r   r  r   putr   r   r6  r@   r   r?   r  r   r   r   ) r4  recaptcha_token	task_typerP  Z
issue_bodyr   ZissueZ	issue_urlr!  r"  r}   Zresponse_messager   Zresponse_statusr&  r.  Zfile_contentr  	timestampZ	unique_idZunique_filenamer   r   r  r   r   r'  Z	image_urlZcomment_bodyrp  
email_body	html_bodyrf   rf   rg   create_issue  s   

 





rA  c                 C   s   t j d}ddd| dgigdgdd}dd	t j d
}tj|||d}| }d }|dr:|d d d }|S t j d}	d| ||di}
tj|	||
d}|js_td|j	 d|j
 | d }|S )Nz/objects/contacts/searchfiltersr   ZEQ)ZpropertyNameoperatorre   r^   )ZfilterGroups
propertiesry  r   Bearer r   r  r1  rt  r   rl  z/objects/contactsrD  )r   	firstnamelastnamez!Hubspot contact creation failed:  )rF   HUBSPOT_V3_APIHUBSPOT_API_TOKENr   r6  r   r   r   r   r   r  )r   rG  rH  Z
search_urlZsearch_bodyr   Zsearch_resultsZsearch_data
contact_idZcreate_contact_urlZcontact_bodyZ
create_resrf   rf   rg   get_hubspot_contact3  sB   
rM  c           	      C   sl   ddt j d}t j d}d| |||di}tj|||d}|js.td|j d	|j |	 d
 }|S )Nr   rE  rF  z/objects/dealsrD  )ZdealnameZ	dealstagepipelineZlead_source_in_dealr1  zHubspot deal creation failed: rI  rl  )
rF   rK  rJ  r   r6  r   r   r   r  r   )	r   ZstagerN  Zlead_sourcer   Zcreate_deal_urlZ	deal_bodyZdeal_resdeal_idrf   rf   rg   create_hubspot_deal_  s    	rP  c                 C   s  ddt j d}t j d}tt  d }d| |di}tj|||d}|j	s6t
d	|j d
|j | d }t j d| d| d}	tj|	|d}
|
j	s^t
d|
j d
|
j t j d| d| d}tj||d}|j	st
d|j d
|j |S )Nr   rE  rF  z/objects/notesi  rD  )Zhs_note_bodyhs_timestampr1  zHubSpot note creation failed: rI  rl  z/objects/notes/z/associations/deals/z/note_to_dealr  z"Failed to associate note to deal: /associations/contacts/z/note_to_contactz%Failed to associate note to contact: )rF   rK  rJ  r  r    utcnowr>  r   r6  r   r   r   r  r   r;  )r  rO  rL  r   Znote_urlrQ  Znote_payloadZnote_resnote_idZassociate_note_to_deal_urlZassociate_res_dealZassociate_note_to_contact_urlZassociate_res_contactrf   rf   rg   create_hubspot_notet  s.   rU  c                 C   s\   ddt j d}t j d|  d| d}tj||d}|js*td|j d	|j |	 S )
Nr   rE  rF  z/objects/deals/rR  z/deal_to_contactr  z,HubSpot deal to contact association failed: rI  )
rF   rK  rJ  r   r;  r   r   r   r  r   )rO  rL  r   Zassociate_urlZ	assoc_resrf   rf   rg   #associate_hubspot_deal_with_contact  s   rV  z/submit_data_inquiryc               
   C   s  t j} t jd}tjd s|rt|stddidfS | dd }| dd }| d	d }| d
d}| dd}| d }| d }| dd}	|r^|r^|r^|r^|sftddidfS |dvrutdd| idfS d }
d }d }|dkrtj	ntj
}|dkrtjntj}|dkrtjnd }i }zt|||}
W n ty } ztdt|ddfW  Y d }~S d }~ww z	t||||}W n ty } ztd|
t|ddfW  Y d }~S d }~ww zt||
 W n  ty } ztd|
|t|ddfW  Y d }~S d }~ww zt|||
}W n ty4 } zd|
|t|d}W Y d }~nd }~ww dd|
||d }|rd!}d}|	dkrRt||d"}n|dkr_t||d"nt||d"}t|}z"ttj||| |d#dd$ |d#< |r|d%dd$ |d%< W n6 ty } z)|r|d%dd& |d%< |d'dt| |d'< n	d(|
|t|d}W Y d }~nd }~ww |r|| t|d)fS t|d*fS )+Nr'  r(  r{   r)  r   r   r`   rG  rH  r  Z
isAnbcFormr  rP  r  ZisServiceFormz4Missing title, body, email, first name, or last name)researchZinterestr-  rW  r  z&Failed to create or retrieve contact. )r{   detailsr  zFailed to create deal. )r{   rL  rX  z'Failed to associate deal with contact. )r{   rL  rO  rX  z:Request successfully submitted, but note creation failed. )r  rL  rO  rX  z Request successfully submitted. r  )r   r   rL  rO  rT  "SPARC Form Submission Confirmationr   r   r   z.Confirmation email sent to user successfully. r  r3  rX  zQRequest successfully submitted, but confirmation email sent to user unsuccessful.   r  )r%   r4  r   r5  r\   r  r$   r6  rF   ZHUBSPOT_ONBOARDING_PIPELINE_IDZ HUBSPOT_GRANT_SEEKER_PIPELINE_IDZ,HUBSPOT_ONBOARDING_PIPELINE_INITIAL_STAGE_IDZ.HUBSPOT_GRANT_SEEKER_PIPELINE_INITIAL_STAGE_IDZANBC_LEAD_SOURCErM  r   r|   rP  rV  rU  rC   r   rB   rA   r  r   r   r   update)r4  r<  r   rG  rH  r=  Zis_anbc_formrP  r  Zis_service_formrL  rO  rT  Zdeal_pipelineZ
deal_stageZdeal_lead_sourceZpartial_successr}   r   rp  r?  r@  rf   rf   rg   submit_data_inquiry  s   

*

r]  z/tasksc               
   C   s  t j} d| v rCz#tjtjtj| d dd}| }d|vs"|d s)ddidfW S W n# tyB } zt	
d| W Y d }~nd }~ww tjd	 sNddidfS d
}t }d }dt jv rrd}t jd }t }t||t|t|j }t }	| d }
|r|r|d r|
d|d  7 }
t|	| d d d d d d |
grd| v r| d r| d }| d p|}d}t||
d}d}| rd| v r| d }|dkrd}n|dkrd}n|dkrd}n|dkrd}t|dkr|r|rttj|||d d! d"|r|id#fS did#fS dd$id%fS )&Nr'  r  r  r  r{   r  r  r  r(  Fr.  Tr   ZwebViewLinkz

Attachment: rP  Z	userEmailZ	firstNamezSPARC SubmissionrZ  r`   r  newszSPARC News SubmissioneventzSPARC Event SubmissionZtoolsAndResourceszSPARC Tool/Resource SubmissionZcommunitySpotlightzSPARC Story Submissionr   
z<br>attachment_filenamer   zFailed registering user datar  )r%   r4  r   r6  rF   r  r  r   r   r   r{   r5  r\   r  r9  r&  r   r   r|   rV   r7  r   r   rA   r   r  r   r   r   r!  )r4  r  r  r  Zhas_attachmentr  r   fileZdrive_clientclientr   Z
user_emailr   rp  r  r=  rf   rf   rg   report_form_submission  sn   


rd  z#/hubspot_contact_properties/<email>c              
   C   s4  t j d|  d}ddt j d}zDtj||d}|jdkr%t| W S |jdkr8td	d
|  dddfW S tdd|j d|jddkrL| n|j	d|jfW S  tj
yx } ztdd|  dt|ddfW  Y d }~S d }~w ty } ztdd|  dt|ddfW  Y d }~S d }~ww )Nz/objects/contacts/zZ?archived=false&idProperty=email&properties=firstname,lastname,email,newsletter,event_namer   rE  rF  r  r   rz   zContact not foundzNo contact with the email 'z' was found in HubSpot.r}  zFailed to fetch contactz'HubSpot API responded with status code .r   )r{   r   rX  r   z"Could not get contact with email 'z' due to a request error.r  zInternal Server ErrorzDAn unexpected error occurred while fetching the contact with email 'z'.)rF   rJ  rK  r   r   r   r$   r   r   r  r   r|   r   )r   r   r   r   r  rf   rf   rg   get_hubspot_contact_propertiesG  sR   




rf  z/subscribe_to_newsletterc               
   C   s  t j} | d}| d}| d}|stddidfS d}z$t|\}}|dkr2|d	 d
d }ntd|  td| W n ty_ } ztd| d|  W Y d }~nd }~ww g }t|t	rs|
d}ttd |}d|vr||d d|||d|d|ddgi}	tj d}
ddtj d}ztj|
|	|d}|  t| dfW S  tjjy } ztdt	|idfW  Y d }~S d }~ww )Nemail_address
first_name	last_namer{   zEmail is requiredr   r`   r   rD  
newsletterz"Unexpected response from HubSpot: zUnexpected error: z4Error while retrieving contact properties for email : ;Z
NewsletterZinputs)r   rG  rH  rj  r   )rD  rl  Z
idPropertyz/objects/contacts/batch/upsertr   rE  rF  r  r  )r%   r   r   r$   rf  r   r{   r   
isinstancer|   splitlistfilterr]  joinrF   rJ  rK  r   r6  r   r   r   )r   r   rh  ri  Znewsletter_propertyZcontact_propertiesr   r}   Zcurrent_newsletter_valuesrw  r   r   r   rf   rf   rg   subscribe_to_newsletterp  s^   


"


 rr  c              
   C   s  t jjtjd}z|jjjjt	| g ddd}W n t
y4 } ztdd| dW  Y d }~S d }~ww |s=tdddS |jsFtdd	dS |jd
sRtdddS |jd
d j}|jdi gd }|rj|jnd}|jdi gd }|r{|jnd}|jd}	t|	dkr|	d }	|	r|	jdng }
|jd}t|dkr|d }|r|jdng }dd |
| D }||||dS )N)Zaccess_token)rG  rH  r   rj  
event_nameF)rL  properties_with_historyZarchivedr   z2Exception thrown when getting contact properties: r   z-Failed to retrieve contact data from HubSpot.zContact properties not foundr   z Contact Email property not foundr   rG  r`   rH  rj  rl  rs  c                 S   s   g | ]}|r|qS rf   rf   r  tagrf   rf   rg   r        z*get_contact_properties.<locals>.<listcomp>)r   rG  rH  tags)hubspotZClientcreaterF   rK  ZcrmZcontactsZ	basic_apiZ	get_by_idr|   r   r#   rt  r   re   r  rn  )	object_idrc  contact_datar}   r   Zfirstname_datarG  Zlastname_datarH  Znewsletter_tags_dataZnewsletter_tagsZevents_tags_dataZevents_tagsrx  rf   rf   rg   get_contact_properties  sD   
r}  c                 C   s   d|  d}ddt j d}|||d||d}z$tj|||d}	t|	jd	kr8td
| d|	j d|	j  |		 W S  t
yi }
 z!td| d|  |
 tdd| d|  d|
 dW  Y d }
~
S d }
~
ww )Nz#https://api.emailoctopus.com/lists/z	/contactsr   rE  rF  )Z	FirstNameZLastName)rg  rB  r   rx  r  Z200z:Emailoctopus contact did not get added/updated for email: z. Returned a response of rk  z4Could not add or update contact with email address: z in emailoctopus list: r  z1Could not add/update contact with email address: z! from emailoctopus list with ID:  due to the following error: r   )rF   ZEMAIL_OCTOPUS_API_KEYr   r;  r|   r   r   r{   r  r   r   r#   )Zlist_idr   rG  rH  rx  r   r   r   rw  r   r  rf   rf   rg   "add_or_update_emailoctopus_contact  s&    
*r  z/hubspot_webhookc               
   C   s  d } zt jdd} W n" ty, } ztd|  tddidfW  Y d }~S d }~ww t| tr4| sDtd|   tddidfS tj	
d	t   tj	
d
|   dt jvs`dt jvrutdt j  tddt j idfS t jd}t jd }zt|}W n ty   td|  tddidf Y S w zftt }|| dkrtd| d|  tddidfW S t j}d}tj| dd}| | | | }	ttjd|	dtj }
t|
d}t||std tddidfW S W n& ty, } ztd|  tdd| idfW  Y d }~S d }~ww dd }| D ]}t|tsEt d|  q3t!"|| q3tdd d!d"fS )#NT)forcezInvalid JSON body: r{   zInvalid JSON formatr   z%Expected an array of webhook events: zExpected a non-empty JSON arrayz"Received Hubspot webhook request: zHubspot webhook request body: zX-HubSpot-Request-TimestampzX-HubSpot-Signature-V3zKRequired signature header(s) not present in the following request headers: z$Invalid signature timestamp format: z"Invalid signature timestamp formatr[   z<Signature timestamp is older than 5 minutes: current time = z, signature time = z+Signature timestamp is older than 5 minutesr   ),:)
separatorsr0  zSignature is invalidi  zBInternal error when validating Hubspot webhook request signature: r  c                 S   s  t   | d}| d}|d u s|d u r(td|   	 W d    d S d }zt|}W n$ tyT } ztd| d|  W Y d }~W d    d S d }~ww zh|d }|d }|d }ttj	|||g d	}|d
krg }	|d D ]}
|
|d vr|	
|
 qvg }|d D ]}
|
|d vr|
|
 qdd |	D }|dd |D  ttj	||||d	 ntd|  W n ty } ztd|  d|  W Y d }~nd }~ww W d    d S W d    d S 1 sw   Y  d S )NsubscriptionTypeobjectIdz Missing required keys in event: z/Could not retrieve contact information for ID: r~  rG  rH  r   Z
subscribedzcontact.propertyChangerx  c                 S      i | ]}|d qS )Trf   ru  rf   rf   rg   
<dictcomp>=      z:hubspot_webhook.<locals>.process_event.<locals>.<dictcomp>c                 S   r  Frf   ru  rf   rf   rg   r  >  r  zUnsupported subscription type: zError processing event rk  )r5  Zapp_contextr   r   r{   r}  r   r  rF   ZEMAIL_OCTOPUS_MASTER_LIST_IDr]  r\  )r_  Zsubscription_typer{  r|  r  rG  rH  r   Zemailoctopus_contactZtags_to_addrv  Ztags_to_removeZupdated_contact_tagsrf   rf   rg   process_event   sZ   






"""z&hubspot_webhook.<locals>.process_eventzSkipping non-dict event: r  z/Webhook request received and signature verifiedr  r   )#r%   r  r   r   r{   r$   rm  ro  r5  loggerr   r   r   r  r  timer   r   r   hmacnewrF   HUBSPOT_CLIENT_SECRETencodehashlibsha256digestr   r   r  Zcompare_digestdictr  executorsubmit)r  r}   Zsignature_headerZtimestamp_headerZsignature_timestampcurrent_timer   r  Zstringified_bodyZ
raw_stringZhashed_signatureZbase64_hashed_signaturer  r  r_  rf   rf   rg   hubspot_webhook  sr   


"%r  z/get-organ-curies/c               
   C   s   t jd} t| }i }ztjtj dtj |d}t	|
 }W t|S  tyD } ztd| ddddfW  Y d }~S d }~ww )	Nrq  r4  r5  zFailed getting Uberon IDsr~  BaseExceptionr  r   )r%   r   rA  r6   r   r6  rF   r7  r8  rO   r   r  r   r{   r$   )rq  ZrequestBodyrm  r   r  rf   rf   rg   get_available_uberonidsO  s&   r  z/get-related-terms/<query>c              
   C   s   t jjdddt jjdddt jjdddtjd}i }ztjtj d	|  |d
}t| }W t|S  t	yV } zt
d| | ddddfW  Y d }~S d }~ww )N	directionZOUTGOING)r  relationshipTypezBFO:0000050entailr  )r  r  r  r   z/graph/neighbors/)r   z*Failed getting related terms with payload r~  r  r  r   )r%   r   r   rF   r8  r   ZSCI_CRUNCH_SCIGRAPH_HOSTrQ   r   r  r   r{   r$   )rF  rw  rm  r   r  rf   rf   rg   get_related_termsg  s,   r  z /simulation_ui_file/<identifier>c                 C   s   t tt| }t|j}z2|d d }|d }|d d d d }tdd| d	| }td
d|}ttt	||W S  t
yP   tddd Y d S w )Nrt  r   Zs3urizabi-simulation-filerY  r   zs3://[^/]*/r`   zfiles/z	s3://|/.*rz   z$no simulation UI file could be foundr   )rL   rD  r4   r   r   r   resubr$   r2  r   r#   )rT  rt  Zresults_jsonre  r#  r   r   Zs3_bucket_namerf   rf   rg   simulation_ui_file  s   r  z	/pmr_filec                  C   sz   t  } | r5d| v r5zttj d| d  }|jdkr$t|j	W S |
 W S    tddd Y d S tddd d S )Nr   r   r   r   zinvalid pathr   zmissing path)r%   r  r   r6  rF   ZPMR_HOSTr   r   r   r   r   r#   )r   r  rf   rf   rg   pmr_file  s   

r  z/start_simulationc                  C   sJ   t  } | rd| v rd| d v rd| d v rtt| S tddd d S )Nsolverr   r   r   z)Missing solver name and/or solver versionr   )r%   r  r   r   do_start_simulationr#   r   rf   rf   rg   rW     s   $rW   z/check_simulationc                  C   sR   t  } | r!d| v r!d| v r!d| d v r!d| d v r!tt| S tddd d S )NZjob_idr  r   r   r   z1Missing solver name, solver version and/or job idr   )r%   r  r   r   do_check_simulationr#   r  rf   rf   rg   rX     s   ,rX   z/pmr_latest_exposurec                  C   s   t  } | rKd| v rKz4tj| d ddid}|jdkr:z| d d d d	 d d
 }W n   d}Y t|dW S | W S    tddd Y d S tddd d S )NZworkspace_urlr  z$application/vnd.physiome.pmr2.json.1r  r   
collectionitemsr   ZlinksZhrefr`   r   r   zInvalid workspace URLr   zMissing workspace URL)r%   r  r   r   r   r   r$   r#   )r   r  r   rf   rf   rg   pmr_latest_exposure  s$   

$
r  z/onto_term_lookupc               
   C   s   t jd} ddi}dtji}t| }z3tjtj d|||d}| }|d d }|d d }|d	kr@|d
 }|d }	|	W S ddi}	|	W S  t	ya }
 zt
d|
 W Y d }
~
tdS d }
~
ww )Nr|  r  r   r   rv  r   r   r   rg  totalr^   r   rh  label	not found.An error occured while fetching from SciCrunchr  )r%   r   r   rF   r8  r7   r   ZSCI_CRUNCH_INTERLEX_HOSTr   r   r   r{   r#   )r|  r   r   rF  r   rt  rg  r  rm  r  r  rf   rf   rg   find_by_onto_term  s,   r  z/dataset_citations/<dataset_id>c              
   C   s   ddi}dt ji}t| }z5tjt j d|||d}| }|d d }|d d d }|d	kr<|d
 }|d }	|	W S ddi}	|	W S  tyZ }
 zt	d|
 W Y d }
~
nd }
~
ww t
dd|  didfS )Nr  r   r   rv  r  rg  r  re   r^   r   rh  z
dataset idr  r  r   z:An error occured while fetching citation info for dataset z from SciCrunchr  )rF   r8  r=   r   r   SCI_CRUNCH_CITATIONS_HOSTr   r   r   r{   r$   )r  r   r   rF  r   rt  rg  r  rm  r  r  rf   rf   rg   get_dataset_citations  s*   r  z/total_dataset_citationsc               
   C   s   ddi} dt ji}dddi idddd	iiid
}z+tjt j d| ||d}| }|d d d }tdd |D }td|idfW S  ty[ } zt	
d| W Y d }~nd }~ww tddddfS )Nr  r   r   r   Z	match_allZ	Citationsr  r=  zcitations.type)r?  r@  rF  r  rv  r  r  rs  c                 s   s    | ]}|d  V  qdS )Z	doc_countNrf   )r  r  rf   rf   rg   r  (	  s    z.get_total_dataset_citations.<locals>.<genexpr>total_citationsr   z>An error occured while fetching total citations from SciCrunchr   )r  r   r  )rF   r8  r   r   r  r   sumr$   r   r   r{   )r   r   rF  r   rt  rs  r  r  rf   rf   rg   get_total_dataset_citations	  s2   r  z/search-readme/<query>c              
   C   st   d|  }ddt j i}ztj||d}| W S  tjjy9 } zt| t	|dddfW  Y d }~S d }~ww )Nz2https://dash.readme.com/api/v1/docs/search?search=r  zBasic )r   r   z9Readme is not currently reachable, please try again laterr}  r   )
rF   ZREADME_API_KEYr   r6  r   r   r   r   r{   r|   )rF  r   r   r   r   rf   rf   rg   search_readme.	  s"   

r  z/metricsc                   C   s   t S rb   )r   rf   rf   rf   rg   metricsA	  s   r  z/event_updatedc                  C   sr   t jd} | tjkrtdd|  d d S t  }|r1zt|W S    tdd| d Y d S tddd d S )NZevent_updated_secret_keyr+  zInvalid secret key: r   r   zInvalid event data: zMissing event data)r%   r   r   rF   ZCTF_CDA_ACCESS_TOKENr#   r  r   )Z
secret_keyr_  rf   rf   rg   event_updatedG	  s   

r  iQ z/all_dataset_idsc                  C   "   t  } dd | D }d}||S )Nc                 S      g | ]}t |qS rf   r|   r  elementrf   rf   rg   r  [	  rw  z#all_dataset_ids.<locals>.<listcomp>, )r
   rq  ro  Zstring_list	delimiterrf   rf   rg   all_dataset_idsW	     
r  z/all_dataset_uuidsc                  C   r  )Nc                 S   r  rf   r  r  rf   rf   rg   r  c	  rw  z%all_dataset_uuids.<locals>.<listcomp>r  )r   rq  r  rf   rf   rg   all_dataset_uuids_	  r  r  z/total_protocol_views   c                  C   sF   t t} | d u s| dstd dddfS | d}td|idfS )Nr   zTotal views not yet calculated.)total_viewsr      r  r   )r   protocolMetricsTabler   r$   )Ztable_stater   rf   rf   rg   get_total_protocol_viewsg	  s   
r  z/contact_supportc               
   C   s   t  } | d}| d}| d}| dd}|r|r|s'tddidfS |rVzttj||t	d|itj
 W n tyU } ztdd	id
fW  Y d }~S d }~ww tddidfS )Nr   r   r   rp  rY  r{   zMissing required fieldsr   z-Confirmation email sent to user unsuccessful.r  zMessage received successfully.r   )r%   r  r   r$   r   r   rF   r   r?   r   ZSERVICES_EMAILr   )r   r   r   r   rp  r}   rf   rf   rg   contact_supportt	  s(   


r  )rx  r   r  )NNrb   (1  atexitZapp.metrics.pennsiever   Zapp.metrics.contentfulr   r   r   Z!scripts.update_contentful_entriesr   r   Zapp.metrics.algoliar   r	   r
   r   r   Zapp.metrics.gar   r   r   r   r   r   Z"scripts.update_featured_dataset_idr   r   Zscripts.update_protocol_metricsr   r   Zapp.osparc.servicesr   r/  r  Zboto3r  r  r   r  ry  Zhubspot.crm.contactsr   r   r   r  r   r  urllib.parser   Z!apscheduler.schedulers.backgroundr   Zapscheduler.triggers.combiningr   Zapscheduler.triggers.dater   Zapscheduler.triggers.intervalr   Zbotocore.exceptionsr   concurrent.futuresr   r    r!   Zflaskr"   r#   r$   r%   Z
flask_corsr&   Zflask_marshmallowr'   ri  r(   Zpennsieve2.directr)   Zpennsieve.baser*   r   ZPILr+   Zrequests.authr,   Zflask_cachingr-   Zapp.scicrunch_requestsr.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   Zscripts.email_senderr>   r?   r@   rA   rB   rC   	threadingrD   Z	xml.etreerE   
app.configrF   Zapp.dbtablerG   rH   rI   rJ   rK   Zapp.scicrunch_process_resultsrL   rM   rN   rO   rP   rQ   rR   Zapp.serializerrS   Zapp.utilitiesrT   rU   rV   Zapp.osparc.osparcrW   r  rX   r  Zapp.biolucida_process_resultsr  rY   r  basicConfigru   r5  Z	LOG_LEVELupper	log_levelr  setLevelgetattrWARNINGr  Z
DEPLOY_ENVr\   cacheZmar   r   rc  ZSPARC_PORTAL_AWS_KEYZSPARC_PORTAL_AWS_SECRETr]   r  ZDATABASE_URLZdb_url
startswithr!  r  AttributeErrorr  r  r   r  objectr_   Zerrorhandlerr~   Zbefore_first_requestr   r   r   r   r   r   r   r   Zprotocol_metricsr  r   r   r   r   Zannotation_cleanup_schedulerZadd_jobZremoveExpiredStater   r   r   r   r   r   r   r   r   Zfeatured_dataset_id_triggerr   registerZrouter   r   r   ZDEFAULT_S3_BUCKET_NAMEr   r   r  r  r%  r)  r0  r2  r:  r<  rC  rG  rJ  rM  rO  rQ  rV  rW  rX  r^  rk  ra  rn  rr  ru  rD  r{  r  r  r  r  r   r  r  r  r  r  r  r  cachedr  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r	  r
  r  r&  rA  rM  rP  rU  rV  r]  rd  rf  rr  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rf   rf   rf   rg   <module>   s    H $




&










>	
%








.

,


0



	

	



:$

g,"g;(;'
e

		
 
