o
    hƒ                 
   @   s  d dl Z d dlmZ d dlmZmZmZ d dlmZm	Z	 d dl
mZmZmZmZmZ d dlmZmZmZmZmZmZ d dlmZmZ d dlmZmZ d d	lmZ d dlZd dl Z d dl!Z!d dl"Z"d dl#Z#d dl$Z$d dl%Z%d dl&Z&d d
l'm(Z( d dl)Z)d dl*Z*d dl+Z+d dl,Z,d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m;Z;m<Z< d dl=m>Z>m?Z?m@Z@mAZA d dlBmCZC d dlDmEZE d dlFmGZG d dlHmIZJ d dlKmLZL d dlMmNZN d dlOmPZP d dlQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZb d dlcmdZdmeZemfZfmgZgmhZhmiZi d dljmkZk d dllmmZm d dlnmoZo d d lpmqZqmrZrmsZsmtZtmuZu d d!lvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~ d d"lmZ d d#lmZmZmZ d d$lmZ d d%lmZ d d&lmwZmZ d dlZe*  e>eZeoj Zejee*ee*j e:d'd(Zeojejd)< ePed*d+d,d-ZeCe eEeZed Zdae!jd.eojeojd/d0Zek ZeojZered1red1d2d3Zzedu rdneqeZW n ey   dZY nw zedu rdnereZW n ey   dZY nw zedu rdneseZW n ey'   dZY nw zedu r0dneteZW n eyA   dZY nw zedu rJdneueZW n ey[   dZY nw G d4d5 d5eZed6d7d8 Zejd9d: Zejd;d< Ze0 Ze0 Ze0 Ze0 Ze0 Ze0 ZeeZeojd=kredu sed>d?kre  ejse*d@ e  ejse*dA e  ere0 Ze   ejejdBdCdCdDdDd d dE eojdFkreojd=krejse*dG e  ejedBdHdIdJ i ZejdKdL Zi Ze Ze Ze ZejdMdN Ze ZejdOdP ZejedQd3dR ejedQd3dR ejedQdSdT e2e4 e6d3dUgZedVdW eΡ eojd=krZejedBdXd d dY dZd[ Ze eϡ ed\d]d^ Zejd_d`gdadbdc Zddde ZedfdgeojfdhdiZedjdkdl ZedmeojfdndoZedpdqdr ZedseojfdtduZdvdw Zedxdydz Zd{d| Zed}eojfd~dZeddd Zededdd Zeddd Zeddd Zededdd Zededdd Zededdd Zeddd Zeddd Zeddd Zeddd Zdd Zdd Zdd Zeddd ZedeojfddZeddd Zdd Zejdddd ddeddyddZejdddideddd Zedáddń ZddǄ ZddɄ Zdd˄ Zed̡dd΄ Zedϡddф ZedҡddԄ Zedաddׄ Zedءddڄ Zejddgdaddބ Zejddgdadd Zejddgdaejd+ddd Zejddgdadd Z dd Zejddgdadd Zejddgdadd Zejddgdadd Zejddgdadd Zejddgdadd ZejddgdadzddZejddgdadd  Zejѐddgdadd Z	ejѐddgdadd Z
ejѐddgdadd	 Zd
d Zdd Zdd Zejѐdd`gdadd Zejѐdd`gdadd Zejѐdd`gdadd Zejѐdd`gdadd Zejѐdd`gdadd Zejѐdd`gdad d! Zd"d# Zd{d$d%Zejѐd&d`gdad'd( Zd)d* Zd|d+d,Zd-d. Zd/d0 Zejѐd1d`gdad2d3 Zejѐd4d`gdad5d6 Zejѐd7dgdad8d9 Zejѐd:d`gdad;d< Zd=d> Z d?d@ Z!ejѐdAd`gdadBdC Z"eѐdDdEdF Z#eѐdGdHdI Z$eѐdJdKdL Z%ejѐdMd`gdadNdO Z&ejѐdPd`gdadQdR ZejѐdSd`gdadTdU ZejѐdVd`gdadWdX Z'eѐdYdZd[ Z(ejѐd\dgdad]d^ Z)ejѐd_dgdad`da Z*ejѐdbdgdadcdd Z+ejѐdedgdadfdg Z,ejѐdhd`gdadidj Z-ejdkdejѐdldgdadmdn Z.ejdkdejѐdodgdadpdq Z/eѐdrejdsddtdu Z0ejѐdvd`gdadwdx Z1dS (}      N)get_download_count)init_cf_cda_clientget_funded_projects_countget_featured_datasets)update_all_events_sort_orderupdate_event_sort_order)get_dataset_countinit_algolia_clientget_all_dataset_idsget_all_dataset_uuidsget_associated_datasets)init_ga_reportingget_ga_1year_sessionsinit_gspread_clientappend_contactupload_fileinit_drive_client)set_featured_dataset_id#get_featured_dataset_id_table_state)update_protocol_metrics get_protocol_metrics_table_state)OSparcServices)ApiException)urlparse)BackgroundScheduler)	OrTrigger)DateTrigger)IntervalTrigger)ClientError)ThreadPoolExecutor)datetime	timedelta)Flaskabortjsonifyrequest)CORS)Marshmallow)	Pennsieve)UnauthorizedException)
new_client)Image)Cache)create_doi_querycreate_filter_requestcreate_facet_querycreate_doi_aggregatecreate_title_querycreate_identifier_query!create_pennsieve_identifier_querycreate_field_querycreate_request_body_for_curiescreate_onto_term_querycreate_multiple_doi_query create_multiple_discoverId_querycreate_anatomy_queryget_body_scaffold_dataset_idcreate_multiple_mimetype_querycreate_citations_querycreate_dataset_flatmap_query)EmailSenderfeedback_emailissue_reporting_email#creation_request_confirmation_email-anbc_form_creation_request_confirmation_email2service_form_submission_request_confirmation_email)Lock)ElementTree)Config)AnnotationTableMapTableScaffoldTableFeaturedDatasetIdSelectorTableProtocolMetricsTable)process_resultsprocess_get_first_scaffold_inforeform_aggregation_resultsreform_curies_resultsreform_dataset_resultsreform_related_termsreform_anatomy_resultsreform_flatmap_query_result)ContactRequestSchema)img_to_base64_strget_path_from_mangled_listget_extension)start_simulation)check_simulation)rL   process_result   )Zmax_workersZENVZSimpleCache,  )Z
CACHE_TYPEZCACHE_DEFAULT_TIMEOUT)configs3z	us-east-1)Zaws_access_key_idZaws_secret_access_keyZregion_namezpostgres://zpostgresql://   c                   @   s`   e Zd ZdZe ed ZdZe	dd Z
dd Ze	dd	 Zd
d Ze	dd Zdd ZdS )	Biolucida i?B Fc                 C   
   | t _d S N)r`   _tokenvalue rg   3/home/cmiss/Jenkins/workspace/SPARC-API/app/main.py	set_token      
zBiolucida.set_tokenc                 C      | j S rc   )rd   selfrg   rg   rh   token      zBiolucida.tokenc                 C   rb   rc   )r`   _expiry_datere   rg   rg   rh   set_expiry_date   rj   zBiolucida.set_expiry_datec                 C   rk   rc   )rp   rl   rg   rg   rh   expiry_date   ro   zBiolucida.expiry_datec                 C   rb   rc   )r`   _pending_authenticationre   rg   rg   rh   set_pending_authentication   rj   z$Biolucida.set_pending_authenticationc                 C   rk   rc   )rs   rl   rg   rg   rh   pending_authentication   ro   z Biolucida.pending_authenticationN)__name__
__module____qualname__rd   r    nowr!   rp   rs   staticmethodri   rn   rq   rr   rt   ru   rg   rg   rg   rh   r`   |   s    


r`     c                 C   s   t t| ddfS )N)errorr{   )r$   str)erg   rg   rh   resource_not_found   s   r   c               
   C   s   zt tjtjdtjdaW d S  tjjy- }  zt	
d t	
|  W Y d } ~ d S d } ~ w tyI }  zt	
d t	
|  W Y d } ~ d S d } ~ w tye }  zt	
d t	
|  W Y d } ~ d S d } ~ ww )NF)Z	api_token
api_secretZenv_overridehostz#Unable to connect to Pennsieve hostz&Unable to authorise with Pennsieve ApizUnknown Error)r(   rF   PENNSIEVE_API_TOKENPENNSIEVE_API_SECRETPENNSIEVE_API_HOSTpsrequests
exceptions	HTTPErrorloggingr|   PSUnauthorizedException	Exceptionerrrg   rg   rh   connect_to_pennsieve   s(   


r   c               
   C   sT   zt tjtjtjd daW d S  ty) }  ztd|   W Y d } ~ d S d } ~ ww )N)api_keyr   Zapi_hostZ	api2_hostz'Error connecting to pennsieve 2 agent: )	r*   rF   r   r   r   ps2r   r   r|   r   rg   rg   rh   connect_to_pennsieve2   s   r   ZFALSEtotal_protocol_viewsz3Starting scheduler for protocol metrics acquisitionz6Starting scheduler for featured dataset id acquisitionZcron*2)yearmonthdayhourminutesecondZdevelopmentz8Starting scheduler for updating contentful event entries   z
US/Eastern)r   timezonec               
   C   s   t d z&tjtj dd} | jr*d| jddv r*|  }t	|d }|t
d< W n tyC } zt d	| W Y d }~nd }~ww tjsRt d
 t  d S d S )NzGetting oSPARC viewersz/viewersurlapplication/jsonzcontent-typera   datafile_viewersz!Could not retreive oSPARC viewersz1Starting scheduler for oSPARC viewers acquisition)r   infor   getrF   OSPARC_API_HOSTokheadersjsonbuild_filetypes_tableosparc_datar   r|   viewers_schedulerrunningstart)reqZviewerstabler~   rg   rg   rh   get_osparc_file_viewers   s    

r   c                  C   st   t d trtt} | td< trtt}|td< tr#tt}|td< t	 }|td< t
js8t d t
  d S d S )NzGathering metrics dataZ1year_sessions_countZdataset_countZfunded_projects_countZ1year_download_countz*Starting scheduler for metrics acquisition)r   r   google_analyticsr   usage_metricsalgoliar   
contentfulr   r   metrics_schedulerr   r   )Zga_responseZalgolia_responseZcf_responseZps_responserg   rg   rh   get_metrics   s    

r   c               
   C   sn   t d ztjtj dd} |  }t|d  W d S  t	y6 } zt 
d| W Y d }~d S d }~ww )NzFetching oSPARC servicesz	/servicesr   r   z%Request to get oSPARC services failed)r   r   r   r   rF   r   r   osparc_servicesZset_servicesr   r|   )r   Zservices_respr~   rg   rg   rh   get_services  s   
r   interval)functriggerdays   )r   r   hours)r   c                   C      t tS rc   )r   featuredDatasetIdSelectorTablerg   rg   rg   rh   <lambda>(  s    r   sat)Zday_of_weekr   r   c                   C   s   t d tjrt  t d tjrt  t d tjr$t  t d tjr0t  t d tjr<t  t d t	jrJt	  d S d S )Nz1Stopping scheduler for oSPARC viewers acquisitionz*Stopping scheduler for metrics acquisitionz2Stopping scheduler for updating contentful entriesz3Stopping scheduler for updating featured dataset idz0Stopping scheduler for updating protocol metricsz&Stopping scheduler for oSPARC services)
r   r   r   r   shutdownr   )update_contentful_event_entries_schedulerfeatured_dataset_id_schedulerprotocol_metrics_schedulerservices_schedulerrg   rg   rg   rh   shutdown_schedulers.  s&   





r   z/healthc                   C   s   t ddiS )NstatusZhealthy)r   dumpsrg   rg   rg   rh   healthF  s   r   z/contactPOST)methodsc               	   C   sh   t tj} t | }|d }|d }|d }t||| tt	j
|dtd|i t ddiS )NnameemailmessagezFeedback submissionr   sent)r   loadsr%   r   rT   loademail_senderZ
send_emailmailersend_emailrF   
SES_SENDERr?   
substituter   )r   Zcontact_requestr   r   r   rg   rg   rh   contactK  s   r   c                 C   s   t jd| |d|d|d}|S )N
get_object	requester)BucketKeyRequestPayerZResponseContentType)ZParamsZ	ExpiresIn)r^   Zgenerate_presigned_url)s3BucketNamekeycontent_type
expirationresponserg   rg   rh   create_s3_presigned_urlZ  s   r   z	/downloadi  c                 C   s6   t jd}t jd|}t jdd}t|||| S )Nr   r   ZcontentTypezapplication/octet-stream)r%   argsr   r   )r   bucket_namer   r   r   rg   rg   rh   create_presigned_urle  s   r   z/thumbnail/neurolucidac               
   C   s  t j} d| vsd| vsd| vrtdddS tj d}z)tj|| dd	}|  |jd
kr=|j	dddkr=t
|jW S tdd W d S  tjjyU   tddd Y S  tjjyd   tdd Y S  tjjy } ztddt| W  Y d }~S d }~ww )NversionZ	datasetIdpath  Query arguments are not valid.descriptionz
/thumbnail   paramstimeout   Content-Typeunknownz	image/pngzFailed to retrieve thumbnail.z0Unable to make a connection to NEUROLUCIDA_HOST.  z&Request to NEUROLUCIDA_HOST timed out.  z)Error while requesting NEUROLUCIDA_HOST: )r%   r   r#   rF   ZNEUROLUCIDA_HOSTr   r   raise_for_statusstatus_coder   base64	b64encodecontentr   ConnectionErrorTimeoutRequestExceptionr}   )
query_argsr   r   r~   rg   rg   rh   thumbnail_from_neurolucida_filen  s&   
 r   z/thumbnail/segmentationc              
   C   s  t j}d|vrtdddS |d| }|d }d}d}d}d}d	}|}	|r'|sztj||d
| d|	 dd}
W n5 tym } z)|jd d dkrYtdd| ddW  Y d}~S tdd| ddW  Y d}~S d}~ww |
d  	d}d|v }d|v }|r|s|	|7 }	n||7 }|	|7 }	t
||k rtdd| ddS |r'|r'|du rtdd| ddS ||dd }|d|d d }t|}|j}d}|D ]}||jdd 7 }qt|}tdt|d t|d f|}t|}|S )z
    Extract a thumbnail from a mbf xml file.
    First phase is to find the thumbnail element in the xml document.
    Second phase is to convert the xml to a base64 png.
    r   r   r   r   r   NFr   i  zbytes=-r   )r   r   ZRanger   ErrorCode	NoSuchKeyr{   Could not find file: ''Unknown error for file: 'BodyzUTF-8z<thumbnail z</thumbnail>z#Could not find thumbnail in file: 'ra   r   ZRGBrowscols)r%   r   r#   r   r^   r   r   r   readdecodelenfindrE   
fromstringattribtextbytesfromhexr+   	frombytesintrU   )r   r   r   r   resourceZstart_tag_foundZend_tag_foundZ
start_byteoffsetZend_byter   exZstart_thumbnail_elementZthumbnail_xmlxmlZ	size_infoZim_datachildZbyte_im_dataZimZbase64_formrg   rg   rh   extract_thumbnail_from_xml_file  s`   
  


"r  z/flatmap/findc                     sP  t j} d| vsd| vrtdddS | d  | d }| dd}ztjtj d|d	d
}|  |jdkr|	 }dtj
i} fdd|d D }g }|D ]x}d|d  }	t|	}
z#tjtj d||
d}|  |	 }t|||	}|r||| W qM tjjy   tddd Y   W S  tjjy   tddd Y   W S  tjjy } ztddt| dW  Y d}~  W S d}~ww t|dkrtdd| d  ddW S t|W S tdd W dS  tjjy   tddd Y S  tjjy   tdd d Y S  tjjy' } ztdd!t| dW  Y d}~S d}~ww )"z
    Find an associated flatmap for a subject given the subject id and dataset id.
    The subject id and dataset id should be in the form:
        subject: sub-f006
        dataset: 2a3d01c0-39d3-464a-8746-54c9d67ebe0f
    datasetsubjectr   r   r   true)instr  zinclude-equivalentz/inst   r   r   r   c                    s   g | ]
}|d   kr|qS )r  rg   ).0itemZtarget_datasetrg   rh   
<listcomp>  s    z7find_associated_flatmap_for_subject.<locals>.<listcomp>resultz
N:dataset:/_search)r   r   z/Unable to make a connection to SCI_CRUNCH_HOST.r   z%Request to SCI_CRUNCH_HOST timed out.r   z1Error while making a request to SCI_CRUNCH_HOST: Nr   r{   zNo results for subject 'z' in dataset ''.z(Failed to retrieve information from QDB.z3Unable to make a connection to SCI_CRUNCH_QDB_HOST.z)Request to SCI_CRUNCH_QDB_HOST timed out.z5Error while making a request to SCI_CRUNCH_QDB_HOST: )r%   r   r#   r   r   rF   ZSCI_CRUNCH_QDB_HOSTr   r   r   KNOWLEDGEBASE_KEYr=   postSCI_CRUNCH_HOSTrS   appendr   r   r   r   r}   r  r$   )r   Ztarget_subjectr   Zqdb_responser   Zsci_crunch_paramsZfiltered_resultsresultsr$  
dataset_idZsci_crunch_queryZknowledge_base_responseZflatmap_dataZassociated_flatmap_infor~   rg   r"  rh   #find_associated_flatmap_for_subject  sb   

(
"r-  z/exists/<path:path>c                 C   sf   t j}|d|}z
tj|| dd}W n ty    ddi Y S w |dd}|dkr/ddiS ddiS )	Nr   r   r   r   r   existsfalseContentLengthr   r  )r%   r   r   r^   head_objectr   )r   r   r   r   head_responsecontent_lengthrg   rg   rh   
url_exists  s   
r5  c                 C   s0   |  dd} | d | d } t| }| S )Nzhttps://api.pennsieve.io/z"https://api.pennsieve.io/discover//)replacerfindr   r   r   )urirrg   rg   rh   fetch_discover_file_information  s   
r;  z/s3-resource/discover_pathc               
   C   s   t jd} zt| }d|v r |d dkr |d d }|d W S W n ty9 } ztd| W Y d }~nd }~ww tdd	|  d
S )Nr9  Z
totalCountr_   filesr   r   zFailed to retrieve uri {uri}r{   zFailed to retrieve uri r   )r%   r   r   r;  r   r   r|   r#   )r9  json_response	file_infor  rg   rg   rh   get_discover_path!  s   
r?  c              
   C   s   z&t j|| dd}|dtj}|r!|tjk s$tdd| dW S W dS W dS  tjjyk } z7|j	d d d	krAdW  Y d }~S |j	d d dkrRdW  Y d }~S t|j	d d |j	d d W  Y d }~S d }~ww )Nr   r.  r1  i  zFile too big to download: r   r  r  Z404r{   .Provided path was not found on the s3 resourceZ403  EThere is a permission issue when accessing the file at specified pathMessage)r   OK)r{   r@  )rA  rB  )
r^   r2  r   rF   ZDIRECT_DOWNLOAD_LIMITr#   botocorer   r   r   )r   r   r3  r4  r   rg   rg   rh   s3_header_check/  s*   *rF  z/s3-resource/<path:path>c           
      C   s   t j}|d|}| }t| |}|d dks|d dkrQt| }||kr*tddd t||}|d dkr8|}n|d dkrEtddd n|d dkrQtddd tj||d	d
}t jd}|d  }	|d urnt	
|	S |	S )Nr   r   r{   rA  r@  r   r   rB  r   r.  ZencodeBase64r  )r%   r   r   rF  rV   r#   r^   r   r
  r   r   )
r   r   r   r   Zs3_pathr   Zs3_path_modifiedZ	response2encode_base64r  rg   rg   rh   direct_download_urlI  s2   


rH  z /scicrunch-dataset/<doi1>/<doi2>c              
   C   s   |  ddd | }t|}ztjtj dtj |d}| W S  tjj	yA } zt
| td|iW  Y d }~S d }~ww )NDOI:ra   r6  /_search?api_key=r   r|   )r7  r-   r   r(  rF   r)  r'  r   r   r   r   r|   r   )Zdoi1Zdoi2doir   r   r   rg   rg   rh   sci_doio  s   

rM  z/pubmed/<id_>z/pubmed/<id_>/c              
   C   s\   zt d|  d}|jW S  t jjy- } zt| td|iW  Y d }~S d }~ww )Nz https://pubmed.ncbi.nlm.nih.gov/r6  r|   )	r   r   r  r   r   r   r|   r   r   )id_r   r   rg   rg   rh   pubmed  s   
rO  z/scicrunch-query-string/c               
   C   s   t jd} t jd}t jd}t jd}t| |||}ztjtj dtj	 |d}t
| W S  tjjyT } zt| td|iW  Y d }~S d }~ww )NfieldcuriesizefromrJ  rK  r|   )r%   r   getlistr   r4   r   r(  rF   r)  r'  rL   r   r   r   r   r|   r   )fieldsrQ  rR  Zfrom_r   r   r   rg   rg   rh   	sci_organ  s    
rV  z/dataset_info/using_doic                  C   s<   t jd} t jd}t| }|d u rtt|S t|S )NrL  Zraw_response)r%   r   r   r-   rP   dataset_search)rL  rawqueryrg   rg   rh   get_dataset_info_doi  s   rZ  z!/dataset_info/using_multiple_doisz"/dataset_info/using_multiple_dois/c                  C       t jd} t| }tt|S )Ndois)r%   r   rT  r7   rL   rW  )r\  rY  rg   rg   rh   get_dataset_info_dois     r]  z./multiple_dataset_info/using_multiple_mimetypez//multiple_dataset_info/using_multiple_mimetype/c                  C   r[  )Nq)r%   r   rT  r;   rL   rW  )r_  rY  rg   rg   rh   get_file_info_from_mimetype  s   r`  z(/dataset_info/using_multiple_discoverIdsz)/dataset_info/using_multiple_discoverIds/c                  C   r[  )NdiscoverIds)r%   r   rT  r8   rL   rW  )ra  rY  rg   rg   rh   get_dataset_info_discoverIds  r^  rb  z/dataset_info/using_titlec                  C   r[  )Ntitle)r%   r   r   r1   rP   rW  )rc  rY  rg   rg   rh   get_dataset_info_title     rd  z%/dataset_info/using_object_identifierc                  C   r[  N
identifier)r%   r   r   r2   rP   rW  rg  rY  rg   rg   rh   "get_dataset_info_object_identifier  re  ri  z/dataset_info/anatomyc                  C   s6   t jdd} | dkrtdddS t| }tt|S )Nrg  r   r{   z Identifier for API call not set.r   )r%   r   r   r#   r9   rR   rW  rh  rg   rg   rh   get_dataset_info_anatomy  s
   rj  z(/dataset_info/using_pennsieve_identifierc                  C   r[  rf  )r%   r   r   r3   rP   rW  rh  rg   rg   rh   %get_dataset_info_pennsieve_identifier  re  rk  c              
   C   s  g }|D ]}|r| d|ks|r| di  dd|kr| di }| di  d}	| di }
|
 d}|
 d}| d	i  d}|rX|| |d
 |d d ||d |	d ur|	D ]7}|dd}t| ||d ||}t|dkr{|| q^|d u s|d u r| |d
 |d d |d}|| q^|d ur|d urtt|D ]%}d|  krt|k rn q|| dd}|td |d || | qq|S )Nrg  r  r   ra   dataciteZisDerivedFromZderived_from_datasetr9  Zassociated_flatmapr   )
discoverIdr   r   r   flatmapUUIDz'derivative/sub-f006/derivative/sub-f006zderivative/sub-f006r   )rm  r   r   r   zhttps://doi.org/)r   r*  r7  +get_is_derived_from_with_identifier_or_pathr  extendrangeget_original_source)rm  r   objectsrg  matchingPathdatasetCacheZsource_listr!  rl  ZisDerivedFromPathsZderivedfromdatasetZderivedfromdatasetdoiZderivedfromdatasetpathrn  r   Zsource_objectsr   irL  rg   rg   rh   ro    sT   


	

ro  c                 C   s   |  di  dg }t|dkrR|d  di  d}|d  di  di  d}|d  di  di  di  d}|d urR|d urR|d urRt||||||S g S )	Nhitsr_   r   _sourcers  	pennsieverg  r   )r   r  ro  )dataset_inforg  rt  ru  rw  rs  rm  r   rg   rg   rh   get_original_source_in_dataset  s   &r{  c           	      C   s   d }| }| d urt | }n!|d ur.|d ur#|}|dd}t|g}n|d ur.|}t|g}||d }|d u r@t|}|||< t|| ||S )NrI  ra   )r2   r7  r7   r8   r   rW  r{  )	rg  rL  rm  r   ru  rY  idZnewDOIrz  rg   rg   rh   rr  #  s"   

rr  z/file_info/get_original_sourcec                  C   sH   t jd} t jd}t jd}t jd}i }dt||| ||iS )Nrm  rL  rg  r   r$  )r%   r   r   rr  )rm  rL  rg  r   ru  rg   rg   rh   get_file_info_original_source7  s   r}  z/segmentation_info/c              
   C   sL  t j}d|vrtdddS |d| }|d}zt||}|d dkr*tddd tj||d	d
}W n5 tyh } z)|jd d dkrTtdd| ddW  Y d }~S tdd| ddW  Y d }~S d }~ww |d 	 }t
|}|d}i }	|d ur|j|	d< n	ddddd|	d< |d}
|
d ur|
j|	d< |	S ddi|	d< |	S )Ndataset_pathr   z,Query arguments must include 'dataset_path'.r   r   r   r{   r@  r   r.  r  r  r  r  r  r  r  z./{*}sparcdata/{*}subjectr  ra   )ZageZsexspeciesZ	subjectidz./{*}sparcdata/{*}atlasZatlasZorgan)r%   r   r#   r   rF  r^   r   r   r   r
  rE   r  r  r  )r   r   r   r~  r   r  r  r  Zsubject_elementr   Zatlas_elementrg   rg   rh   get_segmentation_info_from_fileB  sB   


  



r  z/current_doi_listc                  C   sB   t  } tt| }g }|d d D ]}||d d  qd|iS )NrL  bucketsr   rQ  r+  )r0   rN   rW  r*  )rY  r+  Zdoi_resultsr$  rg   rg   rh   get_all_doio  s   r  c              
   C   sp   z| }dt ji}tjt j d||d}| W S  tjjy7 } zt	| t
d|iW  Y d }~S d }~ww )Nr   r%  )r   r   r|   )rF   r'  r   r(  r)  r   r   r   r   r|   r$   )rY  payloadr   r   r   rg   rg   rh   rW  z  s   

rW  z/search/ra   
   )rY  limitr   )defaultsz/search/<query>c                 C   s   zFt jdd urt jd}t jdd urt jd} t jdd ur+t jd}ttj d|  d| d| dtj 	}t| W S  tj	j
yf } zt| td|iW  Y d }~S d }~ww )	Nr  rY  r   z/_search?q=z&size=z&from=z	&api_key=r|   )r%   r   r   r   rF   r)  r'  rL   r   r   r   r   r|   r   )rY  r  r   r   r   rg   rg   rh   	kb_search  s   *
r  z/filter-search/rY  z/filter-search/<query>/c           	   
   C   s   t jd}t jd}t jd}t jd}t| ||||}ztjtj dtj	 |d}t
| }W |S  tjjy[ } zt| tt|ddd	fW  Y d }~S d }~w tjym   td
ddd	f Y S w )NtermZfacetrR  r   rJ  rK  z<SciCrunch is not currently reachable, please try again laterr|   r   r   8Could not parse SciCrunch output, please try again laterJSONDecodeErrorr   r|   )r%   r   rT  r   r.   r   r(  rF   r)  r'  rL   r   r   r   r   r|   r$   r}   r  )	rY  termsZfacetsrR  r   r   r   r+  r   rg   rg   rh   filter_search  s.   
"r  z/get-facets/<type_>c           
      C   s   t | \}}g }||  D ]U}||d |   d d< ztjtj dtj |d}| }|| W q tjyG   t	dddd	f Y   S  t
ya } ztd
| | W Y d }~qd }~ww g }|D ]}	||	d |   d 7 }qft	|S )Naggregationsr  rP  rJ  rK  r  r  r  r   z$Could not search SciCrunch for path r  )r/   r   r(  rF   r)  r'  r   r*  r  r$   r   r   r|   )
type_Ztype_mapr   r+  r   r   Zjson_resultr  r  r$  rg   rg   rh   
get_facets  s2   r  c                 C   s*   d| v rt | d}|j| d< d S d S )NZreadmemarkdown)r   r   r  )respZmark_reqrg   rg   rh   inject_markdown  s   r  c           
      C   s@  |  d}|  d}|d u s|d u rd S t|}|j}ztj|d|dd}W n> tyf   tjdk}|r;t	
d ztj|d|dd}W n tyc } z|rWt	| W Y d }~Y d S d }~ww Y nw |d	  }zt|}	W n ty } zt	| W Y d }~d S d }~ww |	 d
|	 d|	 dd| d< d S )Nr|  r9  z{}/files/template.jsonr   r.  ZTRUEzXRequired file template.json was not found under /files folder, trying under /packages...z{}/packages/template.jsonr  uuidr   r   )r  r   r   Zstudy)r   r   netlocr^   r   formatr   rF   SPARC_API_DEBUGGINGr   warningr|   r
  r   r   
ValueError)
r  rN  r9  Z
parsed_uribucketr   	debuggingr~   templateZtemplate_jsonrg   rg   rh   inject_template_data  sX   






r  c                 C   sF   i }| D ]}|d   }|d= ||dsg ||< || | q|S )N	file_typeF)lowerr   r*  )Zosparc_viewersr   ZviewerZfiletyperg   rg   rh   r     s   r   z/sim/dataset/<id_>c              
   C   s   t jdkrGz tdtj| }|jr$| }t	| t
| t|W S W n ty@ } ztd|  | W Y d }~nd }~ww tdddS d S )NGETz{}/datasets/{}Could not fetch SIM dataset r{   Resource not foundr   r%   methodr   r   r  rF   DISCOVER_API_HOSTr   r   r  r  r$   r   r   r|   r#   )rN  r   	json_datar  rg   rg   rh   sim_dataset  s   

r  z&/sim/dataset/<id_>/versions/<version_>c              
   C   s   t jdkrKz!tdtj| |}|jr%| }t	| t
| t|W S W n tyD } ztd|  d| | W Y d }~nd }~ww tdddS d S )Nr  z{}/datasets/{}/versions/{}r  z	 version r{   r  r   r  )rN  Zversion_r   r  r  rg   rg   rh   sim_dataset_versions(  s   

$r  z/get_osparc_datac                   C   r   rc   )r$   r   rg   rg   rg   rh   get_osparc_data7     r  z/sim/servicec                  C   sT   t jdkr(t jd} t jjddtd}t jjddtd}t| ||}t|S d S )Nr  searchr  r   )defaulttypeskipr   )r%   r  r   r   r  r   Zsearch_servicesr$   )r  r  r  r+  rg   rg   rh   osparc_search<  s   
r  z	/sim/filec                  C   s    t jdkrtj} td| iS d S )Nr  r   )r%   r  r   Zfile_extensionsr$   )
extensionsrg   rg   rh   osparc_extensionsF  s   
r  z/project/<project_id>r  c                 C   s4   t | }t|d dkrt|d S tddd d S )Nrw  r   r{   r  r   )r   r  r$   r#   )Z
project_iddatasetsrg   rg   rh   datasets_by_project_idO  s   r  z"/get_featured_datasets_identifiersc                   C   s
   dt  iS )Nidentifiers)r   rg   rg   rg   rh   !get_featured_datasets_identifiersY  rj   r  z/get_featured_dataset)r   c               
   C   s   t td } | dkrd} z!tdtj|  }|d g kr+tdtjd }|W S  tyH } zt	
d|  | W Y d }~nd }~ww tddd	 d S )
Nfeatured_dataset_idr       z{}/datasets?ids={}r  zCould not get featured dataset r{   z,An error occured while fetching the resourcer   )r   r   r   r   r  rF   r  r   r   r   r|   r#   )r  r   r  rg   rg   rh   get_featured_dataset^  s   r  z/reva/subject-idsc               
   C   s   z/t dtj } | d }g }|D ]}|d d dkr%||d d  qtd|dd	fW S  tyR } ztd
|  td
|ddfW  Y d }~S d }~ww )N
/packages/childrenr   packageType
Collectionr   success)r   idsr   z+Error while getting REVA subject id files: r   r     )	r   r   rF   ,REVA_3D_TRACING_PRIMARY_FOLDER_COLLECTION_IDr*  r$   r   r   r|   )primary_folderprimary_childrenZsubject_idsr  r~   rg   rg   rh   getRevaSubjectIdsp  s   r  c              
      sX  zd dt dtj }|s#dtj }t| td|dW S |dg }tfdd	|D d }|d u rJd
 }t| td|dW S t d|d d  }|smd|d d  }t| td|dW S |dg }t fdd	|D d }|d u rd }t| td|dW S t d|d d  }|sd|d d  }t| td|dW S |dg }	tfdd	|	D d }
|
d u rd }t| td|dW S t d|
d d  }|sd|
d d  }t| td|dW S |dg W S  ty+ } zd| }t| td|dW  Y d }~S d }~ww )NZCoordinatesDataZInSitur  zPrimary folder not found: r{   r   r  c                 3   $    | ]}|d  d  kr|V  qdS r   r   Nrg   r   r  
subject_idrg   rh   	<genexpr>     " z5getRevaTracingInSituFolderChildren.<locals>.<genexpr>z)Subject folder not found for subject id: r   r|  z,Subject folder could not be fetched for id: c                 3   r  r  rg   r  )coordinates_folder_namerg   rh   r    r  z.CoordinatesData folder not found for subject: z4CoordinatesData folder could not be fetched for id: c                 3   r  r  rg   r  )in_situ_folder_namerg   rh   r    r  z%InSitu folder not found for subject: z+InSitu folder could not be fetched for id: z2Exception thrown when getting Reva InSitu Folder: r  )	r   r   rF   r  r   r|   r#   nextr   )r  r  msgr  subject_childsubject_foldersubject_childrenZcoordinates_childZcoordinates_folderZcoordinates_childrenZin_situ_childZin_situ_folderr~   rg   )r  r  r  rh   "getRevaTracingInSituFolderChildren  s`   











r  z-/reva/anatomical-landmarks-files/<subject_id>c              
      s  zd t | }t fdd|D d }|d u r1td  d|   td  d|  dd	fW S td
|d d  }|d }g }|D ]_}|d d }|d d }td
| }	|	d }
g }|
D ]6}|d d }td
| d}|d d d }td
| d| d }|t|d d t|d qc|t||d qEtd|ddfW S  t	y } ztd|  td|ddfW  Y d }~S d }~ww )NZAnatomicalLandmarksc                 3   r  r  rg   r  Z anatomical_landmarks_folder_namerg   rh   r    r  z2getRevaAnatomicalLandmarksFiles.<locals>.<genexpr>REVA tracing folder  not found for subject: ERROR folder not found for subject: r  r{   r  r   r|  r  r   /viewr   /files/r   )r   s3Url)r   r<  r  )r   foldersr   z4Error while getting REVA anatomical landmarks files z0Error while getting anatomical landmarks files: r  
r  r  r   r|   r$   r   r   r*  r}   r   )r  in_situ_childrenZanatomical_landmarks_childZanatomical_landmarks_folderZanatomical_landmarks_childrenZanatomical_landmarks_foldersZanatomical_landmark_childZlandmark_folder_nameZlandmark_folder_idZanatomical_landmark_folderZlandmark_childrenZlandmark_filesZlandmark_childZlandmark_file_package_idZlandmark_fileZlandmark_file_idZlandmark_file_presigned_urlr~   rg   r  rh   getRevaAnatomicalLandmarksFiles  s:   "r  z /reva/tracing-files/<subject_id>c              
      s  zd t | }t fdd|D d }|d u r1td  d|   td  d|  dd	fW S td
|d d  }|d }g }|D ]R}td
|d d  }|d }|D ]=}	|	d d }
td
|
 d}|d d d }td
|
 d| d }|t|	d d t|d d t|d qYqEtd|ddfW S  t	y } ztd|  td|ddfW  Y d }~S d }~ww )NZ
VagusNervec                 3   r  r  rg   r  Zvagus_nerve_folder_namerg   rh   r    r  z&getRevaTracingFiles.<locals>.<genexpr>r  r  r  r  r  r{   r  r   r|  r  r  r   r  r   r   )r   regionr  r  r   r<  r   z'Error while getting REVA tracing files z#Error while getting tracing files: r  r  )r  r  Zvagus_nerve_childZvagus_nerve_folderZvagus_nerve_childrenZvagus_tracing_filesZvagus_region_childZvagus_region_folderZvagus_region_childrenZvagus_file_childfile_package_idZ
vagus_fileZvagus_file_idZvagus_file_presigned_urlr~   rg   r  rh   getRevaTracingFiles  s8   &r  z!/reva/micro-ct-files/<subject_id>c              
      s   d zt dtj }|d }tfdd|D d }|d u r8td  tdd d	d
fW S t d|d d  }|d }t fdd|D d }|d u rrtd  d  td  d d	d
fW S t d|d d  }|d }g }	|D ]S}
|
d d }t d| d}|d d d }t d| d| d }|
d d }|
d }|
d d }|d d d }|	t	|t	|t	|t	|t	|d qtd|	ddfW S  t
y } ztd|  td|d	dfW  Y d }~S d }~ww )Nz-MicroCTVisualizationr  r  c                 3   r  r  rg   r  r  rg   rh   r    r  z&getRevaMicroCtFiles.<locals>.<genexpr>z/REVA microCT folder not found with subject id: r  z*MicroCT folder not found with subject id: r  r{   r   r|  c                 3   r  r  rg   r  )"micro_ct_visualization_folder_namerg   rh   r    r  zREVA microCT r  r  r   r  r   r   Zstorager  ZfileType)r   r  r  r  rR  r  r  r   z'Error while getting REVA microCT files z#Error while getting microCT files: r  )r   r   rF   Z*REVA_MICRO_CT_PRIMARY_FOLDER_COLLECTION_IDr  r   r|   r$   r*  r}   r   )r  r  r  r  r  r  Zmicro_ct_childZmicro_ct_visualization_folderZmicro_ct_childrenZmicro_ct_filesZmicro_childr  Zmicro_child_fileZmicro_child_file_idZmicro_file_presigned_url	file_name	file_sizeZpackage_typer  r~   rg   )r  r  rh   getRevaMicroCtFiles  sF   
"r  z/get_owner_email/<int:owner_id>c                    sN   t jj}t jj|} fdd|D }|stddd d S td|d jiS )Nc                    s   g | ]	}|j  kr|qS rg   )Zint_id)r   xowner_idrg   rh   r#    s    z#get_owner_email.<locals>.<listcomp>r{   zOwner not foundr   r   r   )r   Z_apiZ_organizationZorganizationsZget_membersr#   r$   r   )r  orgmembersresrg   r  rh   get_owner_email  s   r  z!/get_body_scaffold_info/<species>c                 C   s:   t | }|rt|}tt|}|r|S tdd|  dS )Nr{   zWhole body info not found for r   )r:   r3   rM   rW  r#   )r  r|  rY  r$  rg   rg   rh   get_body_scaffold_info*  s   r  z/thumbnail/<image_id>Fc              
   C   s   t  }zZt | st  W d    n1 sw   Y  tjd|  }d| i}tjd||d}t	
|j}|dkr[|s[t |d W d    n1 sQw   Y  t| d}|W S  tyx } ztd|  | W Y d }~nd }~ww td	d
S )Nz/thumbnail/{0}rn   r  r   s   eyJzdGF0dXMiOiJBZG1pbiB1c2VyIGF1dGhlbnRpY2F0aW9uIHJlcXVpcmVkIHRvIHZpZXcvZWRpdCB1c2VyIGluZm8uIFlvdSBtYXkgbmVlZCB0byBsb2cgb3V0IGFuZCBsb2cgYmFjayBpbiB0byByZXZlcmlmeSB5b3VyIGNyZWRlbnRpYWxzLiJ9ra   Tz Could not get the thumbnail for r{   z-An error occured while fetching the thumbnail)r`   biolucida_lockrn   authenticate_biolucidarF   BIOLUCIDA_ENDPOINTr  r   r%   r   r   r   ri   thumbnail_by_image_idr   r   r|   r#   )image_idZrecursive_callblr   r   r   encoded_contentr  rg   rg   rh   r  6  s2   

r  z/image/<image_id>c              
   C   sl   t jd|  }ztd|}t| W S  ty0 } zt	d|  | W Y d }~nd }~ww t
ddS )Nz/image/info/{0}r  zCould not get image info for r{   z/An error occured while getting the image's info)rF   r  r  r   r%   process_biolucida_resultr   r   r   r|   r#   )r  r   r   r  rg   rg   rh   image_info_by_image_idV  s   
r  z/image_search/<dataset_id>c              
   C   sj   t jd|  }ztd|}| W S  ty. } ztd|  | W Y d }~nd }~ww ddidfS )Nz%/imagemap/search_dataset/discover/{0}r  z$Could not search images for dataset r|   z3An error occured while searching images for datasetr{   )	rF   r  r  r   r%   r   r   r   r|   )r,  r   r   r  rg   rg   rh   image_search_by_dataset_ida  s   
r  z/image_xmp_info/<image_id>c                 C   sv   t jd|  }ztd|}W n tjjy!   tddd Y S w | }|d dkr2t	|d S tdd	|  dS )
Nz/image/xmpmetadata/{0}r  r   )Unable to make a connection to Biolucida.r   r   r  r   zXMP info not found for )
rF   r  r  r   r%   r   r   r#   r   process_biolucida_resultsr  r   r$  r   rg   rg   rh   image_xmp_infol  s   r   z/image_blv_link/<image_id>c                 C   sz   t jd|  }ztd|}W n tjjy!   tddd Y S w | }|d dkr4t	d|d iS tdd	|  dS )
Nz/image/blv_link/{0}r  r   r  r   r   r  linkzBLV link not found for )
rF   r  r  r   r%   r   r   r#   r   r$   r  rg   rg   rh   image_blv_link{  s   r  c                  C   sh   t  } tjd }tjtjdd}g }i }tjd||||d}|jtjj	kr2|
 }| |d  d S d S )Nz/authenticatera   )usernamepasswordrn   r   )r   r   r<  rn   )r`   rF   r  ZBIOLUCIDA_USERNAMEZBIOLUCIDA_PASSWORDr   r%   r   codesr   r   ri   )r  r   r  r<  r   r   r   rg   rg   rh   r    s   
r  c                 C   s\   d}| r&t  }|rd|v r|d }| ||}td|iS tddd d S tddd d S )	NTstater  r   zState not specifiedr   r{   Database not available)r%   get_jsonZ	pushStater$   r#   )r   Zcommitr  r  r  rg   rg   rh   get_share_link  s   r	  c                 C   s`   | r(t  }|r d|v r |d }| |}|r td| |iS tddd d S tddd d S )Nr  r  r   z#Key missing or did not find a matchr   r{   r  )r%   r  Z	pullStater$   r#   )r   r  r  r  rg   rg   rh   get_saved_state  s   
r
  z/annotation/getshareidc                   C   r   rc   )r	  annotationtablerg   rg   rg   rh   get_annotation_share_link  r  r  z/annotation/getstatec                   C   r   rc   )r
  r  rg   rg   rg   rh   get_annotation_state  r  r  z/map/getshareidc                   C   r   rc   )r	  maptablerg   rg   rg   rh   get_map_share_link  r  r  z/map/getstatec                   C   r   rc   )r
  r  rg   rg   rg   rh   get_map_state  r  r  z/scaffold/getshareidc                   C   r   rc   )r	  scaffoldtablerg   rg   rg   rh   get_scaffold_share_link  r  r  z/scaffold/getstatec                   C   r   rc   )r
  r  rg   rg   rg   rh   get_scaffold_state  r  r  c              
   C   s   z&t jtjtj| dd}| }d|vs|d s ddidfW S |ddW S  ty? } zt	d| W Y d }~d S d }~ww )	NZsecretr   r   r   r  r|   Failed Captcha Validation  F0Could not validate captcha, bypassing validation)
r   r(  rF   TURNSTILE_URLNUXT_TURNSTILE_SECRET_KEYr   r   r   r   r|   )rn   
captchaReqcaptchaRespr  rg   rg   rh   verify_recaptcha  s   r  c           	      C   s   dt j dt j d}dt j dd}| |d}|r||d< |r%||d	< tj|||d
}|jdkrB| }|d |d |d dS td|j	 )Nhttps://api.github.com/repos/r6  z/issuestoken application/vnd.github+jsonAuthorizationAccept)rc  bodylabels	assigneesr   r      html_urlcomments_urlr   )r)  r*  issue_api_urlzGitHub Issue creation failed: )
rF   SPARC_GITHUB_ORGSPARC_ISSUES_GITHUB_REPOSPARC_TECH_LEADS_GITHUB_TOKENr   r(  r   r   r   r  )	rc  r$  r%  r&  r   r   r   r   Zresponse_jsonrg   rg   rh   create_github_issue  s&   

r/  z/create_issuec                
   C   sP  t j} t jd}tjd s|rt|stddidfS | dd}| d}| d	}|r0|s6tdd
d | dd }|dv ryzt	| ||gt
jd}|d }|d }|d }	W n' tyx }
 ztdt|
idfW  Y d }
~
S d }
~
ww tdd| idfS d}d}d}|rd}d}d}t j}|rSd|v rS|d }| }|j}t d}t j}| d| d| }dt
j dt
j d| }dt
j d d!d"}t|d#}d$| |d%}zQtj|||d&}|jd'v r.|  }|d( d) }d*| d+}dt
j d d,}d	|i}tj!|||d-}|jdkr)|d.7 }d}d/}n|d07 }n|d17 }d}d/}W n tyR }
 z|d17 }d}d/}W Y d }
~
nd }
~
ww |rd2}t"#d3|i}|d4krmd5}t$#d3|i}t%%|}zt&'t
j(||| |d67 }W n ty }
 z|d77 }d}d/}W Y d }
~
nd }
~
ww t|||	|d8|fS )9Ncaptcha_tokenTESTINGr|   Invalid reCAPTCHAr   r  bugrc  r$  zMissing title or bodyr   r   ra   )r3  feedbacktest)r%  r&  r)  r*  r+  r  Unsupported task type: zSubmission could not be createdz!Submission created successfully. r(  r  
attachmentz%Y%m%d-%H%M%S_r  r6  z/contents/attachments/r  r   z
2022-11-28)r"  r#  zX-GitHub-Api-Versionutf-8z	Add file )r   r   r   r   )r   r(  r   Zdownload_urlz![Issue Attachment]()r!  r'  zFile attachment unsuccessful. r  zFile attachment successful. zFile upload unsuccessful. zSPARC Reported Issue Submissionr   r4  z"SPARC Reported Feedback Submissionz,Confirmation email sent to user successful. .Confirmation email sent to user unsuccessful. )r   r   r+  r   ))r%   formr   appr]   r  r$   r#   stripr/  rF   ZGITHUB_ISSUE_ASSIGNEESr   r}   r<  r
  filenamer    ry   strftimer  uuid4hexr,  r-  r.  r   r   r  r   putr   r   r(  r@   r   r?   r  r   r   r   ) r=  recaptcha_token	task_typerc  Z
issue_bodyr   ZissueZ	issue_urlr*  r+  r~   Zresponse_messager   Zresponse_statusr<  r7  Zfile_contentr  	timestampZ	unique_idZunique_filenamer   r   r  r   r   r=  Z	image_urlZcomment_bodyr  
email_body	html_bodyrg   rg   rh   create_issue
  s   

 





rJ  c                 C   s   t j d}ddd| dgigdgdd}dd	t j d
}tj|||d}| }d }|dr:|d d d }|S t j d}	d| ||di}
tj|	||
d}|js_td|j	 d|j
 | d }|S )Nz/objects/contacts/searchfiltersr   ZEQ)ZpropertyNameoperatorrf   r_   )ZfilterGroups
propertiesr  r   Bearer r   r"  r:  r+  r   r|  z/objects/contactsrM  )r   	firstnamelastnamez!Hubspot contact creation failed:  )rF   HUBSPOT_V3_APIHUBSPOT_API_TOKENr   r(  r   r   r   r   r   r  )r   rP  rQ  Z
search_urlZsearch_bodyr   Zsearch_resultsZsearch_data
contact_idZcreate_contact_urlZcontact_bodyZ
create_resrg   rg   rh   get_hubspot_contactr  sB   
rV  c           	      C   sl   ddt j d}t j d}d| |||di}tj|||d}|js.td|j d	|j |	 d
 }|S )Nr   rN  rO  z/objects/dealsrM  )ZdealnameZ	dealstagepipelineZlead_source_in_dealr:  zHubspot deal creation failed: rR  r|  )
rF   rT  rS  r   r(  r   r   r   r  r   )	r   ZstagerW  Zlead_sourcer   Zcreate_deal_urlZ	deal_bodyZdeal_resdeal_idrg   rg   rh   create_hubspot_deal  s    	rY  c                 C   s  ddt j d}t j d}tt  d }d| |di}tj|||d}|j	s6t
d	|j d
|j | d }t j d| d| d}	tj|	|d}
|
j	s^t
d|
j d
|
j t j d| d| d}tj||d}|j	st
d|j d
|j |S )Nr   rN  rO  z/objects/notesi  rM  )Zhs_note_bodyhs_timestampr:  zHubSpot note creation failed: rR  r|  z/objects/notes/z/associations/deals/z/note_to_dealr  z"Failed to associate note to deal: /associations/contacts/z/note_to_contactz%Failed to associate note to contact: )rF   rT  rS  r  r    utcnowrG  r   r(  r   r   r   r  r   rD  )r$  rX  rU  r   Znote_urlrZ  Znote_payloadZnote_resnote_idZassociate_note_to_deal_urlZassociate_res_dealZassociate_note_to_contact_urlZassociate_res_contactrg   rg   rh   create_hubspot_note  s.   r^  c                 C   s\   ddt j d}t j d|  d| d}tj||d}|js*td|j d	|j |	 S )
Nr   rN  rO  z/objects/deals/r[  z/deal_to_contactr  z,HubSpot deal to contact association failed: rR  )
rF   rT  rS  r   rD  r   r   r   r  r   )rX  rU  r   Zassociate_urlZ	assoc_resrg   rg   rh   #associate_hubspot_deal_with_contact  s   r_  z/submit_data_inquiryc               
   C   s  t j} t jd}tjd s|rt|stddidfS | dd }| dd }| d	d }| d
d}| dd}| d }| d }| dd}	|r^|r^|r^|r^|sftddidfS |dvrutdd| idfS d }
d }d }|dkrtj	ntj
}|dkrtjntj}|dkrtjnd }i }zt|||}
W n ty } ztdt|ddfW  Y d }~S d }~ww z	t||||}W n ty } ztd|
t|ddfW  Y d }~S d }~ww zt||
 W n  ty } ztd|
|t|ddfW  Y d }~S d }~ww zt|||
}W n ty4 } zd|
|t|d}W Y d }~nd }~ww dd|
||d }|rd!}d}|	dkrRt||d"}n|dkr_t||d"nt||d"}t|}z%tjtj|||tjd# |d$dd% |d$< |r|d&dd% |d&< W n6 ty } z)|r|d&dd' |d&< |d(dt| |d(< n	d)|
|t|d}W Y d }~nd }~ww |r|| t|d*fS t|d+fS ),Nr0  r1  r|   r2  r   r   ra   rP  rQ  r  Z
isAnbcFormr0  rc  r$  ZisServiceFormz4Missing title, body, email, first name, or last name)researchZinterestr6  r`  r  z&Failed to create or retrieve contact. )r|   detailsr  zFailed to create deal. )r|   rU  ra  z'Failed to associate deal with contact. )r|   rU  rX  ra  z:Request successfully submitted, but note creation failed. )r  rU  rX  ra   Request successfully submitted. r  )r   r   rU  rX  r]  "SPARC Form Submission Confirmationr   r   Zccr   z.Confirmation email sent to user successfully. r  r<  ra  zQRequest successfully submitted, but confirmation email sent to user unsuccessful.   r(  )r%   r=  r   r>  r]   r  r$   r?  rF   ZHUBSPOT_ONBOARDING_PIPELINE_IDZ HUBSPOT_GRANT_SEEKER_PIPELINE_IDZ,HUBSPOT_ONBOARDING_PIPELINE_INITIAL_STAGE_IDZ.HUBSPOT_GRANT_SEEKER_PIPELINE_INITIAL_STAGE_IDZANBC_LEAD_SOURCErV  r   r}   rY  r_  r^  rC   r   rB   rA   r  r   r   r   SERVICES_EMAILupdate)r=  rE  r   rP  rQ  rF  Zis_anbc_formrc  r$  Zis_service_formrU  rX  r]  Zdeal_pipelineZ
deal_stageZdeal_lead_sourceZpartial_successr~   r   r  rH  rI  rg   rg   rh   submit_data_inquiry  s   

*

ri  z/tasksc               
   C   sb  t j} d| v rCz#tjtjtj| d dd}| }d|vs"|d s)ddidfW S W n# tyB } zt	
d| W Y d }~nd }~ww tjd	 sNddidfS d
}t }d }d }dddd}d}	dt jv rzd}t jd }t }
t|
|t|t|j }W n' ty } z|d  d7  < d|d< d}	td|  W Y d }~nd }~ww | d }|r|r|d r|d|d  7 }||d< |dddd}zt }t|| dd d d d d |g}W n  ty } ztd|  d d!d"d#fW  Y d }~S d }~ww |sd d!d"d#fS z{| d$}|r| d%p|}d&}t|d'| d(d)}| d*d }|d+kr6d,}n|d-kr>d.}n|d/krFd0}n|d1krMd2}t|d3kr|r|r|rx| }t |!d4}t"#tj$|||||j|j% n	t"&tj$||| W n( ty } z|d  d57  < d|d< d}	td6|  W Y d }~nd }~ww t'||	fS )7Nr0  r  r  r  r|   r  r  r  r1  Frb  ra   )r   r   attachment_filenamer(  r7  Tr   z"Failed to upload attachment file. r  r   rf  z*[ERROR] Failed to upload task attachment: r   ZwebViewLinkz

Attachment: rj  z

z<br>rc  z-[ERROR] Failed to add task to Google Sheets: zRequest submission failedfailure)r   r   r  Z	userEmailZ	firstNamezSPARC Submissionz<div>z</div>rd  r  newszSPARC News SubmissioneventzSPARC Event SubmissionZtoolsAndResourceszSPARC Tool/Resource SubmissionZcommunitySpotlightzSPARC Story Submissionr   r9  r<  z+[ERROR] Failed to send confirmation email: )(r%   r=  r   r(  rF   r  r  r   r   r   r|   r>  r]   r  rB  r<  r   r   r}   rW   r@  printr7  r   r   r   rA   r   r?  r  r
  r   r   r  r   Z mailersend_email_with_attachmentr   Zmimetyper   r$   )r=  r  r  r  Zhas_attachmentr  fileZfile_upload_responserequest_responseZrequest_response_codeZdrive_clientr~   r   clientr  Z
user_emailr   r  r$  rF  Z
file_bytesZencoded_filerg   rg   rh   report_form_submissionJ  s   








rs  z#/hubspot_contact_properties/<email>c              
   C   s4  t j d|  d}ddt j d}zDtj||d}|jdkr%t| W S |jdkr8td	d
|  dddfW S tdd|j d|jddkrL| n|j	d|jfW S  tj
yx } ztdd|  dt|ddfW  Y d }~S d }~w ty } ztdd|  dt|ddfW  Y d }~S d }~ww )Nz/objects/contacts/zZ?archived=false&idProperty=email&properties=firstname,lastname,email,newsletter,event_namer   rN  rO  r  r   r{   zContact not foundzNo contact with the email 'z' was found in HubSpot.r  zFailed to fetch contactz'HubSpot API responded with status code .r   )r|   r   ra  r   z"Could not get contact with email 'z' due to a request error.r  zInternal Server ErrorzDAn unexpected error occurred while fetching the contact with email 'r&  )rF   rS  rT  r   r   r   r$   r   r   r  r   r}   r   )r   r   r   r   r  rg   rg   rh   get_hubspot_contact_properties  sR   




ru  z/subscribe_to_newsletterc               
   C   s  t j} | d}| d}| d}|stddidfS d}z$t|\}}|dkr2|d	 d
d }ntd|  td| W n ty_ } ztd| d|  W Y d }~nd }~ww g }t|t	rs|
d}ttd |}d|vr||d d|||d|d|ddgi}	tj d}
ddtj d}ztj|
|	|d}|  t| dfW S  tjjy } ztdt	|idfW  Y d }~S d }~ww )Nemail_address
first_name	last_namer|   zEmail is requiredr   ra   r   rM  
newsletterz"Unexpected response from HubSpot: zUnexpected error: z4Error while retrieving contact properties for email : ;Z
NewsletterZinputs)r   rP  rQ  ry  r   )rM  r|  Z
idPropertyz/objects/contacts/batch/upsertr   rN  rO  r'  r  )r%   r   r   r$   ru  r   r|   r   
isinstancer}   splitlistfilterr*  joinrF   rS  rT  r   r(  r   r   r   )r   r   rw  rx  Znewsletter_propertyZcontact_propertiesr   r~   Zcurrent_newsletter_valuesr  r   r   r   rg   rg   rh   subscribe_to_newsletter  s^   


"


 r  c              
   C   s  t jjtjd}z|jjjjt	| g ddd}W n t
y4 } ztdd| dW  Y d }~S d }~ww |s=tdddS |jsFtdd	dS |jd
sRtdddS |jd
d j}|jdi gd }|rj|jnd}|jdi gd }|r{|jnd}|jd}	t|	dkr|	d }	|	r|	jdng }
|jd}t|dkr|d }|r|jdng }dd |
| D }||||dS )N)Zaccess_token)rP  rQ  r   ry  
event_nameF)rU  properties_with_historyZarchivedr   z2Exception thrown when getting contact properties: r   z-Failed to retrieve contact data from HubSpot.zContact properties not foundr   z Contact Email property not foundr   rP  ra   rQ  ry  r{  r  c                 S   s   g | ]}|r|qS rg   rg   r   tagrg   rg   rh   r#  /      z*get_contact_properties.<locals>.<listcomp>)r   rP  rQ  tags)hubspotZClientcreaterF   rT  ZcrmZcontactsZ	basic_apiZ	get_by_idr}   r   r#   r  r   rf   r  r}  )	object_idrr  contact_datar~   r   Zfirstname_datarP  Zlastname_datarQ  Znewsletter_tags_dataZnewsletter_tagsZevents_tags_dataZevents_tagsr  rg   rg   rh   get_contact_properties  sD   
r  c                 C   s   d|  d}ddt j d}|||d||d}z$tj|||d}	t|	jd	kr8td
| d|	j d|	j  |		 W S  t
yi }
 z!td| d|  |
 tdd| d|  d|
 dW  Y d }
~
S d }
~
ww )Nz#https://api.emailoctopus.com/lists/z	/contactsr   rN  rO  )Z	FirstNameZLastName)rv  rU  r   r  r'  Z200z:Emailoctopus contact did not get added/updated for email: z. Returned a response of rz  z4Could not add or update contact with email address: z in emailoctopus list: r  z1Could not add/update contact with email address: z! from emailoctopus list with ID:  due to the following error: r   )rF   ZEMAIL_OCTOPUS_API_KEYr   rD  r}   r   r   r|   r  r   r   r#   )Zlist_idr   rP  rQ  r  r   r   r   r  r   r  rg   rg   rh   "add_or_update_emailoctopus_contact8  s&    
*r  z/hubspot_webhookc               
   C   s  d } zt jdd} W n" ty, } ztd|  tddidfW  Y d }~S d }~ww t| tr4| sDtd|   tddidfS tj	
d	t   tj	
d
|   dt jvs`dt jvrutdt j  tddt j idfS t jd}t jd }zt|}W n ty   td|  tddidf Y S w zftt }|| dkrtd| d|  tddidfW S t j}d}tj| dd}| | | | }	ttjd|	dtj }
t|
d}t||std tddidfW S W n& ty, } ztd|  tdd| idfW  Y d }~S d }~ww dd }| D ]}t|tsEt d|  q3t!"|| q3tdd d!d"fS )#NT)forcezInvalid JSON body: r|   zInvalid JSON formatr   z%Expected an array of webhook events: zExpected a non-empty JSON arrayz"Received Hubspot webhook request: zHubspot webhook request body: zX-HubSpot-Request-TimestampzX-HubSpot-Signature-V3zKRequired signature header(s) not present in the following request headers: z$Invalid signature timestamp format: z"Invalid signature timestamp formatr\   z<Signature timestamp is older than 5 minutes: current time = z, signature time = z+Signature timestamp is older than 5 minutesr   ),:)
separatorsr9  zSignature is invalidi  zBInternal error when validating Hubspot webhook request signature: r  c                 S   s  t   | d}| d}|d u s|d u r(td|   	 W d    d S d }zt|}W n$ tyT } ztd| d|  W Y d }~W d    d S d }~ww zh|d }|d }|d }ttj	|||g d	}|d
krg }	|d D ]}
|
|d vr|	
|
 qvg }|d D ]}
|
|d vr|
|
 qdd |	D }|dd |D  ttj	||||d	 ntd|  W n ty } ztd|  d|  W Y d }~nd }~ww W d    d S W d    d S 1 sw   Y  d S )NsubscriptionTypeobjectIdz Missing required keys in event: z/Could not retrieve contact information for ID: r  rP  rQ  r   Z
subscribedzcontact.propertyChanger  c                 S      i | ]}|d qS )Trg   r  rg   rg   rh   
<dictcomp>      z:hubspot_webhook.<locals>.process_event.<locals>.<dictcomp>c                 S   r  Frg   r  rg   rg   rh   r    r  zUnsupported subscription type: zError processing event rz  )r>  Zapp_contextr   r   r|   r  r   r  rF   ZEMAIL_OCTOPUS_MASTER_LIST_IDr*  rh  )rn  Zsubscription_typer  r  r  rP  rQ  r   Zemailoctopus_contactZtags_to_addr  Ztags_to_removeZupdated_contact_tagsrg   rg   rh   process_event  sZ   






"""z&hubspot_webhook.<locals>.process_eventzSkipping non-dict event: r  z/Webhook request received and signature verifiedr  r   )#r%   r  r   r   r|   r$   r|  r~  r>  loggerr   r   r   r  r  timer   r   r   hmacnewrF   HUBSPOT_CLIENT_SECRETencodehashlibsha256digestr   r   r  Zcompare_digestdictr  executorsubmit)r$  r~   Zsignature_headerZtimestamp_headerZsignature_timestampcurrent_timer   r  Zstringified_bodyZ
raw_stringZhashed_signatureZbase64_hashed_signaturer  r  rn  rg   rg   rh   hubspot_webhookN  sr   


"%r  z/get-organ-curies/c               
   C   s   t jd} t| }i }ztjtj dtj |d}t	|
 }W t|S  tyD } ztd| ddddfW  Y d }~S d }~ww )	Nr  rJ  rK  zFailed getting Uberon IDsr  BaseExceptionr  r   )r%   r   rT  r5   r   r(  rF   r)  r'  rO   r   r  r   r|   r$   )r  ZrequestBodyr$  r   r  rg   rg   rh   get_available_uberonids  s&   r  z/get-related-terms/<query>c              
   C   s   t jjdddt jjdddt jjdddtjd}i }ztjtj d	|  |d
}t| }W t|S  t	yV } zt
d| | ddddfW  Y d }~S d }~ww )N	directionZOUTGOING)r  relationshipTypezBFO:0000050entailr  )r  r  r  r   z/graph/neighbors/)r   z*Failed getting related terms with payload r  r  r  r   )r%   r   r   rF   r'  r   ZSCI_CRUNCH_SCIGRAPH_HOSTrQ   r   r  r   r|   r$   )rY  r  r$  r   r  rg   rg   rh   get_related_terms  s,   r  z /simulation_ui_file/<identifier>c                 C   s   t tt| }t|j}z2|d d }|d }|d d d d }tdd| d	| }td
d|}ttt	||W S  t
yP   tddd Y d S w )Nr+  r   Zs3urizabi-simulation-filer  r   zs3://[^/]*/ra   zfiles/z	s3://|/.*r{   z$no simulation UI file could be foundr   )rL   rW  r3   r   r   r   resubr$   rH  r   r#   )rg  r+  Zresults_jsonr!  r9  r   r   Zs3_bucket_namerg   rg   rh   simulation_ui_file  s   r  z	/pmr_filec                  C   sz   t  } | r5d| v r5zttj d| d  }|jdkr$t|j	W S |
 W S    tddd Y d S tddd d S )Nr   r6  r   r   zinvalid pathr   zmissing path)r%   r  r   r(  rF   ZPMR_HOSTr   r   r   r   r   r#   )r   r  rg   rg   rh   pmr_file  s   

r  z/start_simulationc                  C   sJ   t  } | rd| v rd| d v rd| d v rtt| S tddd d S )Nsolverr   r   r   z)Missing solver name and/or solver versionr   )r%   r  r   r   do_start_simulationr#   r   rg   rg   rh   rX   	  s   $rX   z/check_simulationc                  C   sR   t  } | r!d| v r!d| v r!d| d v r!d| d v r!tt| S tddd d S )NZjob_idr  r   r   r   z1Missing solver name, solver version and/or job idr   )r%   r  r   r   do_check_simulationr#   r  rg   rg   rh   rY   	  s   ,rY   z/pmr_latest_exposurec                  C   s   t  } | rKd| v rKz4tj| d ddid}|jdkr:z| d d d d	 d d
 }W n   d}Y t|dW S | W S    tddd Y d S tddd d S )NZworkspace_urlr#  z$application/vnd.physiome.pmr2.json.1r  r   
collectionitemsr   ZlinksZhrefra   r   r   zInvalid workspace URLr   zMissing workspace URL)r%   r  r   r   r   r   r$   r#   )r   r  r   rg   rg   rh   pmr_latest_exposure	  s$   

$
r  z/onto_term_lookupc               
   C   s   t jd} ddi}dtji}t| }z3tjtj d|||d}| }|d d }|d d }|d	kr@|d
 }|d }	|	W S ddi}	|	W S  t	ya }
 zt
d|
 W Y d }
~
tdS d }
~
ww )Nr  r#  r   r   r%  r   r   r   rw  totalr_   r   rx  label	not found.An error occured while fetching from SciCrunchr  )r%   r   r   rF   r'  r6   r   ZSCI_CRUNCH_INTERLEX_HOSTr   r   r   r|   r#   )r  r   r   rY  r   r+  rw  r  r$  r  r  rg   rg   rh   find_by_onto_term5	  s,   r  z/dataset_citations/<dataset_id>c              
   C   s   ddi}dt ji}t| }z5tjt j d|||d}| }|d d }|d d d }|d	kr<|d
 }|d }	|	W S ddi}	|	W S  tyZ }
 zt	d|
 W Y d }
~
nd }
~
ww t
dd|  didfS )Nr#  r   r   r%  r  rw  r  rf   r_   r   rx  z
dataset idr  r  r   z:An error occured while fetching citation info for dataset z from SciCrunchr  )rF   r'  r<   r   r   SCI_CRUNCH_CITATIONS_HOSTr   r   r   r|   r$   )r,  r   r   rY  r   r+  rw  r  r$  r  r  rg   rg   rh   get_dataset_citationsU	  s*   r  z/total_dataset_citationsc               
   C   s   ddi} dt ji}dddi idddd	iiid
}z+tjt j d| ||d}| }|d d d }tdd |D }td|idfW S  ty[ } zt	
d| W Y d }~nd }~ww tddddfS )Nr#  r   r   r   Z	match_allZ	Citationsr  rP  zcitations.type)rR  rS  rY  r  r%  r  r  r  c                 s   s    | ]}|d  V  qdS )Z	doc_countNrg   )r   r  rg   rg   rh   r  	  s    z.get_total_dataset_citations.<locals>.<genexpr>total_citationsr   z>An error occured while fetching total citations from SciCrunchr   )r  r   r  )rF   r'  r   r   r  r   sumr$   r   r   r|   )r   r   rY  r   r+  r  r  r  rg   rg   rh   get_total_dataset_citationsr	  s2   r  z/search-readme/<query>c              
   C   st   d|  }ddt j i}ztj||d}| W S  tjjy9 } zt| t	|dddfW  Y d }~S d }~ww )Nz2https://dash.readme.com/api/v1/docs/search?search=r"  zBasic )r   r   z9Readme is not currently reachable, please try again laterr  r   )
rF   ZREADME_API_KEYr   r(  r   r   r   r   r|   r}   )rY  r   r   r   r   rg   rg   rh   search_readme	  s"   

r  z/metricsc                   C   s   t S rc   )r   rg   rg   rg   rh   metrics	  s   r  z/event_updatedc                  C   sr   t jd} | tjkrtdd|  d d S t  }|r1zt|W S    tdd| d Y d S tddd d S )NZevent_updated_secret_keyrA  zInvalid secret key: r   r   zInvalid event data: zMissing event data)r%   r   r   rF   ZCTF_CDA_ACCESS_TOKENr#   r  r   )Z
secret_keyrn  rg   rg   rh   event_updated	  s   

r  iQ z/all_dataset_idsc                  C   "   t  } dd | D }d}||S )Nc                 S      g | ]}t |qS rg   r}   r   elementrg   rg   rh   r#  	  r  z#all_dataset_ids.<locals>.<listcomp>, )r
   r  r~  Zstring_list	delimiterrg   rg   rh   all_dataset_ids	     
r  z/all_dataset_uuidsc                  C   r  )Nc                 S   r  rg   r  r  rg   rg   rh   r#  	  r  z%all_dataset_uuids.<locals>.<listcomp>r  )r   r  r  rg   rg   rh   all_dataset_uuids	  r  r  z/total_protocol_views   c                  C   sF   t t} | d u s| dstd dddfS | d}td|idfS )Nr   zTotal views not yet calculated.)total_viewsr      r  r   )r   protocolMetricsTabler   r$   )Ztable_stater   rg   rg   rh   get_total_protocol_views	  s   
r  z/contact_supportc               
   C   s   t  } | d}| d}| d}| dd}|r|r|s'tddidfS |rYztjtj||t	d|itj
tj
d	 W n tyX } ztdd
idfW  Y d }~S d }~ww tddidfS )Nr   r   r   r  rc  r|   zMissing required fieldsr   re  z-Confirmation email sent to user unsuccessful.r  zMessage received successfully.r   )r%   r  r   r$   r   r   rF   r   r?   r   rg  r   )r   r   r   r   r  r~   rg   rg   rh   contact_support	  s*   



r  )r  r   r  )NNrc   (2  atexitZapp.metrics.pennsiever   Zapp.metrics.contentfulr   r   r   Z!scripts.update_contentful_entriesr   r   Zapp.metrics.algoliar   r	   r
   r   r   Zapp.metrics.gar   r   r   r   r   r   Z"scripts.update_featured_dataset_idr   r   Zscripts.update_protocol_metricsr   r   Zapp.osparc.servicesr   rE  r  Zboto3r  r  r   r  r  Zhubspot.crm.contactsr   r   r   r  r   urllib.parser   Z!apscheduler.schedulers.backgroundr   Zapscheduler.triggers.combiningr   Zapscheduler.triggers.dater   Zapscheduler.triggers.intervalr   Zbotocore.exceptionsr   concurrent.futuresr   r    r!   Zflaskr"   r#   r$   r%   Z
flask_corsr&   Zflask_marshmallowr'   ry  r(   Zpennsieve.baser)   r   Zpennsieve2.directr*   ZPILr+   Zflask_cachingr,   Zapp.scicrunch_requestsr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   Zscripts.email_senderr>   r?   r@   rA   rB   rC   	threadingrD   Z	xml.etreerE   
app.configrF   Zapp.dbtablerG   rH   rI   rJ   rK   Zapp.scicrunch_process_resultsrL   rM   rN   rO   rP   rQ   rR   rS   Zapp.serializerrT   Zapp.utilitiesrU   rV   rW   Zapp.osparc.osparcrX   r  rY   r  Zapp.biolucida_process_resultsr  rZ   r  r  basicConfigrv   r>  Z	LOG_LEVELupper	log_levelr  setLevelgetattrWARNINGr  Z
DEPLOY_ENVr]   cacheZmar   r   rr  ZSPARC_PORTAL_AWS_KEYZSPARC_PORTAL_AWS_SECRETr^   r  ZDATABASE_URLZdb_url
startswithr7  r  AttributeErrorr  r  r   r  objectr`   Zerrorhandlerr   Zbefore_first_requestr   r   r   r   r   r   r   r   Zprotocol_metricsr  r   r   r   r   Zannotation_cleanup_schedulerZadd_jobZremoveExpiredStater   r   r   r   r   r   r   r   r   Zfeatured_dataset_id_triggerr   registerZrouter   r   r   ZDEFAULT_S3_BUCKET_NAMEr   r   r  r-  r5  r;  r?  rF  rH  rM  rO  rV  rZ  r]  r`  rb  rd  ri  rj  rk  ro  r{  rr  r}  r  r  rW  r  r  r  r  r  r   r  r  r  r  r  r  r  cachedr  r  r  r  r  r  r  r  r  r  r  r   r  r  r	  r
  r  r  r  r  r  r  r  r/  rJ  rV  rY  r^  r_  ri  rs  ru  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rg   rg   rg   rh   <module>   s    L (




&










>
=	
%








.

,


0



	

	


:$

g,"ga(;'
e

		
 
