U cgk@s&ddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z ia iaddddd d iiiid d iaiad ZdZdZddgZddgZdZdZddZddZddZddZddZdd Zd!d"Zd#d$Zd%d&Z d'd(Z!d)d*Z"Gd+d,d,ej#Z$e%d-kr"e&dS).N)urljoin)Config)name_mapbiolucida_name_mapZNote dataset_id biolucida_idZscicrunch_file_pathZimage_name_in_biolucidaZfilename_in_scicrunch_file_pathzyNot all listed name mappings are required. Better to go through the mapping file and check before using in the sparc api.)ZFormatZMessagezGhttps://github.com/ABI-Software/scicrunch-knowledge-testing/tree/doc_v1z$pennsieve-prod-discover-publish-use1z not-specifiedz image/jp2zimage/vnd.ome.xml+jp2z image/jpxzimage/vnd.ome.xml+jpxFcCsHddi}dtji}tjd}dd|iidgd}tjt|d |||d S) Nacceptapplication/jsonapi_key/Ztermzpennsieve.identifier.aggregateobjects.dataset)query_source&_search?preference=abiknowledgetestingjsonparamsheadersrZSCICRUNCH_API_KEYZSCICRUNCH_API_HOSTrequestsZpostr)rrrscicrunch_hostscicrunch_requestrP/home/cmiss/Jenkins/workspace/Biolucida test/tests/slow_tests/biolucida_tests.py%get_dataset_info_pennsieve_identifier3s   rc CsTddi}dtji}tjd}||ddddd d d d d dg d}tjt|d|||dS)Nrr r r z item.curiez item.namez item.typeszobjects.biolucidazobjects.additional_mimetypezobjects.mimetyper zpennsieve.versionzpennsieve.identifierz pennsieve.uri)fromsizerrrr)startrrrrrrrr getDatasetsGs$  rcCs|ddS)Nr )split) original_namerrrextract_bucket_nameasr"cCs|}ts(|tkr(|t|kr(t||}||ddkr||d}|tkr^|dddn|tkrv|ddd|tkrit|<|t|krit||<|t||krit|||<|ddt||||<d |d <|tkrd |d <|SdS) Nr  ScicrunchPath BiolucidaIdz2Conflict between Pennsieve and Biolucida response.zKFILE VIEWER: Biolucida filename does not match with the Pennsieve filename.ReasonDetailz2Conflict between Scicrunch and Biolucida response.zPBIOLUCIDA VIEWER: Biolucida filename does not match with the Scicrunch filename.z?Please check the name mapping file output for more information.NameMappingRequiredTThis is a known inconsistency issue which has been manually mapped in the sparc api.NameMappingSolved)MAPPING_IMPLEMENTATION pathMappingr BIOLUCIDA_2Dupdate BIOLUCIDA_3D nameMappingr) imageName localPathrrmimetypefilePatherror_responserrrtestBiolucidaAndScicrunchds8    r8c Csg}|d|}|tkr$t|}nJttjd|d|d}|}d|krn|d}t|dkrn|t|<t|dkr|D]}dt d|} dt d|d } ||d ks| | kr|t krit |<|d t ||<d d d } |t krd | d<| SqdS)N_ /datasets/ /versions/z /metadatafilesr z [.a-zA-Z0-9]+pathz?Please check the path mapping file output for more information.z6Correct file path is found through Pennsieve metadata.)PathMappingRequiredFurtherr+PathMappingSolved) pennsieveMetadataCachergetrPENNSIEVE_API_HOSTrlenjoinrefindallr.r) rversionfileNamer6Zfiles_metadatakeyZmetadata_responseZ metadata_infoZ file_metadataZmodified_fileNameZmodified_metadata_pathr7rrr compareWithMetadataFromPennsieves0  rLcCslg}|tkrt|}nRtjd|d|d|}t|}|}d|krh|d}t|dkrh|t|<|S)Nr:r;z/files/browse?path=r<r)pennsieveCacherrDrrCrrE)rrI folderPathr<ZfileUrlZ file_responseZ files_inforrrfetchFilesFromPennsieves   rOcCs ||d}|}|ddd}|ddd}t|||}t|dkr|} d} |D]V} | | dkrtd} qqVd| krV| dddd} | rV| } | |krVd} qqV| sd |d <t||||} | r|| |Sn*d |d <t||||} | r|| |SdS) Nr$r rFr>Turiz'File path cannot be found on Pennsieve.r(z)Folder path cannot be found on Pennsieve.)rsplitrOrElowerrLr0)r4rrIrr7r6rNrJr<ZlPathZ foundFileZ localFileZuriFileZcompare_responserrrtestScicrunchAndPennsieves@    rTc CsFg}d}d}|dd}d|kr(d|}tr<|tkrzfiles/z/image/z*Cannot get a valid request from Biolucida.)r%r&r(statuszpermission deniedzBiolucida permission denied.namez(Image name cannot be found on Biolucida.)r-rrrCrBIOLUCIDA_ENDPOINT status_coder BIOLUCIDA_MAPPING_IMPLEMENTATIONrrTappendr8 Exceptionstr)rrIbiolucida_objectrbucketr5Z responsesr4r3biolucida_responseZ image_infoZspErrorZbsErrorerrr testBiolucidasT          rccCs6|dt}|tkr|d}|s2|ddt}|S)NZadditional_mimetyperXr5)rC NOT_SPECIFIED)r_ mime_typerrrget_mimietype_from_object3s   rfc" CsPiag}g}g}g}g}g} g} d} d} d} d}d}ttjd|}|jdkr|}d|kr|ddkrd} dd|d D}|D]}|d t}|tkr|d }|rd} ||krt|}|t ks|t kr|| krd}| |n | |q||krd} | |qi}|D]}|d t}|tkr|d }|r|| krz|| krzt ||||||}|r| |n*|| kr||krg||<|| |qt|D]}g}t||}||D]D}t|}|t ks|t krt ||||||}|r| |qt|d krd }|D]}d |kr.|d7}q.|D]@}d |krd|kr||krxd|d<nd|d<| |qL|tkr|t|krt||=tt|d krt|=q| s| rd}| r| s| ddd| r| r| s| ddd| s6| r6| ddd|rf| ddjdt| dt| ddt|}||d }d }d }d }d } d }!|D]N}d |kr|d7}d!|kr|d7}d|kr| d7} d"|kr|!d7}!q||}|d krBd#|i|d$<|d kr || || d%|d$d&<|d krB||!||!d%|d$d'<||||d(S))NFTz"/imagemap/search_dataset/discover/rVrWsuccesscSsg|]}|ddqS)Zimage_idN)rC).0Zimagerrr Vsz'test_biolucida_list..Zdataset_images biolucida identifierrr*rPr,zONone of the duplicate biolucida metadata is matched with Biolucida information.ZMetadataRequiredzHPlease clean up the unmatched duplicate biolucida metadata on Scicrunch.ZCleanUpRequiredzbOne or more Biolucida ID found on SciCrunch but no image information is found on Biolucida server.zQNo Biolucida images will be displayed. Biolucida server data update may required.r'z_Specific Biolucida ID found on SciCrunch but no image information is found on Biolucida server.zYImage information is found on Biolucida server but no Biolucida ID is found on SciCrunch.zNNo Biolucida images will be displayed. Scicrunch metadata update may required.z+Duplicate image ids are found on Scicrunch.z$Redundant images are found on {ids}.z, )ZidszTIssues may occur on thumbnail or viewer. More detail will be shown in object errors.)r(r)Totalr@rlZObjectsr?rArlZ Inconsistency)rlZ NameMappedZ NameUnMappedNamePath) FileReportsDatasetWarnings DatasetErrorsBiolucidaFound)rMrrCrrYrZrrdrfr/r1r\rcextendlistkeysrEr2formatrFset)"rrI object_listr`ZdatasetWarningsZ datasetErrorsZ objectErrorsZ biolucida_idsZ scicrunch_idsZbipresence_idsZduplicate_biolucidaZbiolucidaObjectFoundZbiolucidaIDMatchZbiolucidaImageFoundZbiolucidaFoundZduplicateFoundraZ dataset_infor_rjrreZduplicate_cacheerrorZduplicateObjectErrorsZduplicate_countZduplicate_error_countZnumberOfErrorsZ fileReportsZnumberOfInconsistencyZnumberOfNameInconsistencyZnumberOfPathInconsistencyZnumberOfNameMappedZnumberOfPathMappedrrrtest_biolucida_list=s                               r{cCsHdd|dggdgdd}d|krD|d}d|kr^|ddd|d <|dd d|d <d |krDd |d krDd|d krD|d d}|d d d}||d<||d<t}d|d krt|d d}|r6d|kr|dng}t||||}|d|d|d|d|d|d<|d|d<n|dd|S)NZnone_idrrm)IdDOIr|WarningsErrors ObjectErrorsritemrXrnZcurier~Z pennsieverIrkr}ZVersionrQZobjectsrrqrrrrprrs BiolucidazMissing version)rCS3_BUCKET_NAMEr"r{rtr\)rUreportsourcerrIr`ryZobject_reportsrrrtest_datasets_informations: &   rcs$eZdZfddZddZZS)BiolucidaDatasetFilesTestcstj||dS)N)super__init__)selfargskwds __class__rrrsz"BiolucidaDatasetFilesTest.__init__c Csd}d}d}d}d}d}d}dddggggd}d} d} |rXt||} |d | j| } |t| d d krvd }||}| d d D]} t| }d |kr|d r| d } td|dt|ddks|dddkr|d|d|d|qt|ddkr|d|d|d|q|t| d d }|| kr8d }q8||d<| |d<td|dt|d|d<t|d|d<td|dtd|d|ddkrtd|d|ddkrtd |dtj tj |dd!t |d"}tj ||d#d$W5QRXtj tj |dd!t |d"}tj t|d#d$W5QRXtj tj |dd!t |d"}tj t|d#d$W5QRXtd%||dt|ddS)&NrTzreports/biolucida_reports.jsonz#reports/biolucida_name_mapping.jsonz#reports/biolucida_path_mapping.json)TestedWarnedFailed WarnedIds FailedIdsWarnedDatasetsFailedDatasetsirVhitsFrrPzReports generated for r}rrrlrrrrrrzTested Datasets with BiolucidazNumber of datasets tested: rrz Number of dataset with warning: zNumber of dataset with errors: zWarned Datasets: zFailed Datasets: )exist_okw)indentz"Full report has been generated at )r assertEqualrZrrErprintr\osmakedirsr>dirnameopendumpr2r.)rrrZ keepGoingZ totalSizeZ reportOutputZnameMappingOutputZpathMappingOutputZreportsZtestSizeZtotalBiolucidaZscicrunch_responsedatarUrZoutfilerrrtest_files_informationsji "   z0BiolucidaDatasetFilesTest.test_files_information)__name__ __module__ __qualname__rr __classcell__rrrrrs r__main__)'ZunittestrrZ urllib.parseZurllibrrGrZ tests.configrZ/tests.slow_tests.manifest_name_to_discover_namerrrMrBr2r.Zdoc_linkrrdr/r1r-r[rrr"r8rLrOrTrcrfr{rTestCaserrmainrrrrsf   ''*D 2%7