Recent updates

pythonScripts/anonymizeSegmentations.py

@@ -0,0 +1,198 @@
+import os
+import json
+import re
+import subprocess
+import nibabel
+import shutil
+import sys
+import datetime
+
+if len(sys.argv)<2:
+    print("Usage {} version(v1 or similar)".format(sys.argv[0]))
+    sys.exit(0)
+
+#sourceDir=sys.argv[1]
+ver=sys.argv[1]
+
+shome=os.path.expanduser('~nixUser')
+fhome=os.path.expanduser('~')
+with open(os.path.join(fhome,".labkey","setup.json")) as f:
+    setup=json.load(f)
+
+sys.path.insert(0,setup["paths"]["labkeyInterface"])
+import labkeyInterface
+import labkeyDatabaseBrowser
+import labkeyFileBrowser
+
+#sys.path.insert(1,shome+'/software/src/IPNUMM/dicomUtils')
+#import loadDicom
+
+#onko
+configOnko=os.path.join(fhome,'.labkey','onko-nix.json')
+netOnko=labkeyInterface.labkeyInterface()
+netOnko.init(configOnko)
+dbOnko=labkeyDatabaseBrowser.labkeyDB(netOnko)
+fbOnko=labkeyFileBrowser.labkeyFileBrowser(netOnko)
+
+
+#merlin
+configMerlin=os.path.join(fhome,'.labkey','merlin.json')
+netMerlin=labkeyInterface.labkeyInterface()
+netMerlin.init(configMerlin)
+dbMerlin=labkeyDatabaseBrowser.labkeyDB(netMerlin)
+fbMerlin=labkeyFileBrowser.labkeyFileBrowser(netMerlin)
+
+projectOnko='iPNUMMretro/Study'
+datasetOnko='Imaging1'
+
+projectMerlin='iPNUMMretro/Study'
+datasetMerlin='Imaging1'
+
+tempBase=os.path.join(fhome,'temp')
+
+#all images from database
+
+#imageSelector={"CT":"CT","PET":"PETWB"};
+imageResampledField={"Segm":"Segmentation"}
+fileCode="Segm"
+
+participantField='PatientId'
+visitCode='visitCode'
+#for prosepective
+#participantField='ParticipantId'
+
+#generate key from known imaging dates
+
+def getImagingDates(db,project,dataset,participant,\
+        participantField='PatientId'):
+    idFilter={'variable':participantField,\
+            'value':participant,
+            'oper':'eq'}
+    ds=db.selectRows(project,'study',dataset,[idFilter])
+    return [datetime.datetime.strptime(row['studyDate'], '%Y/%m/%d %H:%M:%S')\
+            for row in ds['rows']]
+
+def getImagingDatesDictionary(db,project,dataset,\
+        participantField='PatientId'):
+    ds=db.selectRows(project,'study',dataset,[])
+    participants=[row[participantField] for row in ds['rows']]
+    #unique
+    participants=list(set(participants))
+    return {p:getImagingDates(db,project,dataset,p,participantField)\
+            for p in participants}
+
+def imagingDatesMatch(dateList1,dateList2):
+    n=min(len(dateList1),len(dateList2))
+    
+    datesEqual=[abs(d2-d1)<datetime.timedelta(days=1) \
+            for d1,d2 in zip(dateList1[:n],dateList2[:n])]
+
+    return all(datesEqual)
+
+def findMatch(dateList,dateDictionary):
+    for p in dateDictionary:
+        if imagingDatesMatch(dateList,dateDictionary[p]):
+            return p
+    return None
+
+
+#use webdav to transfer file (even though it is localhost)
+
+
+def getPatientLabel(row,participantField='PatientId'):
+    return row[participantField].replace('/','_') 
+
+def getVisitLabel(row):
+    return 'VISIT_'+str(int(row['SequenceNum']))
+
+def getStudyLabel(row,participantField='PatientId'):
+    return getPatientLabel(row,participantField)+'-'+getVisitLabel(row)
+
+def updateRow(db,project,dataset,row,imageResampledField,gzFileNames):
+    for im in imageResampledField:
+        row[imageResampledField[im]]=gzFileNames[im]
+    db.modifyRows('update',project,'study',dataset,[row])
+ 
+dOnko=getImagingDatesDictionary(dbOnko,projectOnko,datasetOnko,\
+        participantField)
+dMerlin=getImagingDatesDictionary(dbMerlin,projectMerlin,datasetMerlin,\
+        participantField)
+
+i=0
+idMatch={d0:findMatch(dOnko[d0],dMerlin) for d0 in dOnko}
+
+for id in idMatch:
+    print("{} : {}".format(id,idMatch[id]))
+
+dsOnko=dbOnko.selectRows(projectOnko,'study',datasetOnko,[])
+
+im=list(imageResampledField.keys())[0]
+
+for rowOnko in dsOnko["rows"]:
+    
+    
+    segFileOnko=rowOnko[imageResampledField[im]]
+    dirOnko=fbOnko.buildPathURL(projectOnko,\
+            ['preprocessedImages',getPatientLabel(rowOnko,participantField),\
+            getVisitLabel(rowOnko)])
+    fileOnko=dirOnko+'/'+segFileOnko
+    
+
+    if not fbOnko.entryExists(fileOnko):
+        print("Original file {} not available".format(fileOnko))
+        continue
+    
+    
+    
+    
+    idFilter={'variable':participantField,\
+            'value':idMatch[rowOnko[participantField]],\
+            'oper':'eq'}
+    visitFilter={'variable':visitCode,
+            'value':rowOnko[visitCode],
+            'oper':'eq'}
+    
+    dsMerlin=dbMerlin.selectRows(projectMerlin,'study',datasetMerlin,\
+            [idFilter,visitFilter])
+    
+    if len(dsMerlin['rows'])==0: 
+        continue
+    
+    rowMerlin=dsMerlin['rows'][0]
+    
+    
+    
+    segFileMerlin=getStudyLabel(rowMerlin,participantField)+'_'+\
+            im+'_'+ver+'.nii.gz'
+    dirMerlin=fbMerlin.buildPathURL(projectMerlin,\
+            ['preprocessedImages',getPatientLabel(rowMerlin,participantField),\
+            getVisitLabel(rowMerlin)])
+    fileMerlin=dirMerlin+'/'+segFileMerlin
+    gzTargetFiles={im:segFileMerlin}
+
+    if fbMerlin.entryExists(fileMerlin):
+        print("Target file {} already uploaded".format(fileMerlin))
+        updateRow(dbMerlin, projectMerlin,datasetMerlin,rowMerlin,\
+                imageResampledField,gzTargetFiles)
+        
+        continue
+
+    #upload file
+    localFile=os.path.join(tempBase,segFileMerlin)
+    print("Local {}".format(localFile))
+    fbOnko.readFileToFile(fileOnko,localFile)
+    fbMerlin.writeFileToFile(localFile,fileMerlin)
+    print("Remote {}".format(fileMerlin))
+    os.remove(localFile)
+
+    #update row and let it know where the processed files are
+    updateRow(dbMerlin, projectMerlin,datasetMerlin,rowMerlin,
+            imageResampledField,gzTargetFiles)
+   
+
+    if i==-1:
+        break
+    i=i+1
+
+
+print("Done")

pythonScripts/populateImagingFromTransferList.py

@@ -0,0 +1,123 @@
+#date sorts studies from orthanc dataset into target study dataset
+
+import os
+import json
+import re
+import sys
+import datetime
+import re
+
+def main(parameterFile):
+    fhome=os.path.expanduser('~')
+    fsetup=os.path.join(fhome,'.labkey','setup.json')
+    with open(fsetup,'r') as f:
+        setup=json.load(f)
+
+    sys.path.insert(0,setup['paths']['labkeyInterface'])
+    import labkeyInterface
+    import labkeyDatabaseBrowser
+    import labkeyFileBrowser
+
+    fconfig=os.path.join(fhome,'.labkey','network.json')
+
+    net=labkeyInterface.labkeyInterface()
+    net.init(fconfig)
+    db=labkeyDatabaseBrowser.labkeyDB(net)
+    fb=labkeyFileBrowser.labkeyFileBrowser(net)
+
+    with open(parameterFile,'r') as f:
+        pars=json.load(f)
+
+
+
+    i=0
+    #from orthancDatabase/Imaging dataset
+    projectOrthanc=pars['Orthanc']['project']
+    inputQuery=pars['Orthanc']['queryName']
+    inputSchema=pars['Orthanc']['schemaName']
+    inputParticipantField=pars['Orthanc']['participantField']
+
+    #to target project dataset
+    projectStudy=pars['Database']['project']
+    #'iPNUMMretro/Study'
+    #for prospective, set
+    #projectStudy='IPNUMMprospektiva/Study'
+    outputQuery=pars['Database']['queryName']
+    outputSchema=pars['Database']['schemaName']
+    #select patientId that are contained in the demographics dataset
+    transferQuery=pars['Database']['transferQuery']
+    dbParticipantField=pars['Database']['participantField']
+
+    #make a list of images
+    dsImage=db.selectRows(projectStudy,outputSchema,transferQuery,[])
+
+    for im in dsImage['rows']:
+        idFilter={'variable':inputParticipantField,'value':im[dbParticipantField],\
+            'oper':'eq'}
+        #have to convert from datetime to %Y%m%d format
+        #dateFilter={'variable':'imageDate','value':im['imageDate'],'oper':'eq'}
+
+        dsOrthanc=db.selectRows(projectOrthanc,inputSchema,inputQuery,[idFilter])
+
+        for im1 in dsOrthanc['rows']:
+
+            date=datetime.datetime.strptime(im1['studyDate'],'%Y/%m/%d %H:%M:%S') 
+            #convert date to %Y%m%d notation
+            dateYMD=date.strftime('%Y%m%d')
+            if dateYMD!=im['imageDate']:
+                print('Rejecting mismatch: {}/{}'.format(dateYMD,im['imageDate']))
+                continue
+        
+            outvar='NONE'
+            sd=im1['seriesDescription']
+            if sd=='PET WB':
+                outvar='PETWB_orthancId'
+                print('Found PET: {}'.format(im1['orthancSeries']))
+            if sd.find('CT WB')==0:
+                if sd.find('fov')<0:
+                    outvar='CT_orthancId'
+                    print('Found CT: {}'.format(im1['orthancSeries']))
+
+            #skip irrelevant series
+            if outvar=='NONE':
+                continue
+
+            #figure out which row in output study to update
+            filters=[]
+            idFilter={'variable':dbParticipantField,'value':im[dbParticipantField],'oper':'eq'}
+            seqNum=im['imagingVisitId']
+            seqFilter={'variable':'SequenceNum','value':str(seqNum),'oper':'eq'}
+            print('Participant {} Sequence number {}'.format(im[dbParticipantField],str(seqNum)))
+            #ds1 are the matching outputs in target dataset
+            ds1=db.selectRows(projectStudy,outputSchema,outputQuery,\
+                [idFilter,seqFilter])
+    
+            if len(ds1['rows'])>1:
+                print('ERROR: too many matches for {}/{}'.\
+                    format(im[dbParticipantField],seqNum))
+                continue
+
+            mode='update'
+            outRow={}
+            if len(ds1['rows'])==0:
+                mode='insert'
+        
+                outRow[dbParticipantField]=im[dbParticipantField]
+                outRow['SequenceNum']=seqNum
+                outRow['dicomStudy']=im1['dicomStudy']
+        
+            else:
+                outRow=ds1['rows'][0]
+        
+            outRow[outvar]=im1['orthancSeries']
+            outRow['studyDate']=im1['studyDate']
+            outRow['imagingVisitId']=im['imagingVisitId']
+            outRow['visitCode']='VISIT_'+str(im['imagingVisitId'])
+
+            status=db.modifyRows(mode,projectStudy,outputSchema,outputQuery,[outRow])
+            print('{}'.format(status))
+        
+    print("Done")
+
+if __name__=='__main__':
+    main(sys.argv[1])