Robustifying for missing images, mismatches, etc.

pythonScripts/generateFigures.py

@@ -46,6 +46,12 @@ def getCOWAxis(seg,val,axis):
         s1=i
         break
     s1+=1
+    try:
+        sm=numpy.average(x,weights=s)
+    except ZeroDivisionError:
+        print('getCOWaxis - Zero division error')
+        raise
+
     return [s0,numpy.average(x,weights=s),s1]
 
 def getGeometry(seg,val):
@@ -160,10 +166,18 @@ def main(parameterFile):
     imageResampledField={"CT":"ctResampled","PET":"petResampled","patientmask":"ROImask"}
 
     rows=ds['rows']
+    #rows=[r for r in rows if r[participantField]=='8701/08']
     #rows=[ds['rows'][0]]
 
     for r in rows:
-        print(r)
+        missingCodes=[r[f]==None for f in ['patientCode','visitCode']]
+        if any(missingCodes):
+            print('[{}/{}] - Skipping, missing codes'.\
+                    format(r[participantField],r['SequenceNum']))
+            continue
+
+
+        #print(r)
         iTypes=['CT','PET','Segm']
         needToCalculate=False
         for t in ['CT','PET']:
@@ -172,7 +186,8 @@ def main(parameterFile):
             verFilter={'variable':'version','value':pars['version'],'oper':'eq'}
             typeFilter={'variable':'type','value':t,'oper':'eq'}
             
-            ds2=db.selectRows(project,reportSchema,reportQuery,[idFilter,visitFilter,verFilter,typeFilter])
+            ds2=db.selectRows(project,reportSchema,reportQuery,\
+                    [idFilter,visitFilter,verFilter,typeFilter])
             
             if len(ds2['rows'])==0:
                 #skip if row is present
@@ -181,15 +196,33 @@ def main(parameterFile):
                 break
 
         if not needToCalculate:
+            print('[{}/{}] - done'.format(r[participantField],r['SequenceNum']))
+            continue
+        
+        ds1=db.selectRows(project,pars['segmentationSchema'],pars['segmentationQuery'],\
+                [idFilter,visitFilter,verFilter])
+              
+        #check if CT, PET and Segm images are set
+        imagesAvailable=[r[imageResampledField[t]] for t in ['CT','PET']]
+        imagesAvailable=[f!=None for f in imagesAvailable]
+        try:
+            imagesAvailable.append(ds1['rows'][0]['segmentation']!=None)
+        except IndexError:
+            imagesAvailable.append(False)
+
+        if not all(imagesAvailable):
+            print('[{}/{}] Skipping - not all images available :{}'.\
+                    format(r[participantField],r['SequenceNum'],imagesAvailable))
             continue
+
+
+
         imgs={} 
         for t in iTypes:
             
             try:
                 imagePath=r['_labkeyurl_'+imageResampledField[t]]
             except KeyError:
-                ds1=db.selectRows(project,pars['segmentationSchema'],pars['segmentationQuery'],\
-                        [idFilter,visitFilter,verFilter])
                 imagePath=ds1['rows'][0]['_labkeyurl_segmentation']
 
             localPath=os.path.join(tempBase,'image'+t+'.nii.gz')
@@ -198,12 +231,24 @@ def main(parameterFile):
             fb.readFileToFile(imagePath,localPath)
             img=nibabel.load(localPath)
             imgs[t]=img.get_fdata()
+
         print('Loading completed')
+
         for t in ['CT','PET']:
             for val in [3,4,5]:
-                outfile=plot(imgs,t,val,tempBase)
-                remoteDir=fb.buildPathURL(project,[pars['imageDir'],r['patientCode'],r['visitCode']])
-                imageFile=r['patientCode']+'-'+r['visitCode']+'_'+t+'_{}'.format(val)+'_'+pars['version']+'.png'
+
+
+                try:
+                    outfile=plot(imgs,t,val,tempBase)
+                except ZeroDivisionError:
+                    print('[{}/{}] - Skipping. Failed to plot for organ [{}]'.\
+                            format(r[participantField],r['SequenceNum'],val))
+                    continue
+                
+                remoteDir=fb.buildPathURL(project,
+                        [pars['imageDir'],r['patientCode'],r['visitCode']])
+                imageFile=r['patientCode']+'-'+r['visitCode']+'_'+t+'_{}'.\
+                        format(val)+'_'+pars['version']+'.png'
                 remoteFile='/'.join([remoteDir,imageFile])
                 fb.writeFileToFile(outfile,remoteFile)
                 print('Uploaded {}'.format(remoteFile))

pythonScripts/organ_percentile.py

@@ -91,8 +91,12 @@ def main(parameterFile):
     #        if r['patientCode']=='NIX-LJU-D2002-IRAE-A010' \
     #        and r['visitCode']=='VISIT_6']
     #rows=rows1
+    #rows=[r for r in rows if r[participantField]=='956/18']
 
     for r in rows:
+
+        #is segmentation available
+
         localPET=os.path.join(tempBase,'PET.nii.gz')
         localSeg=os.path.join(tempBase,'Seg.nii.gz')
         for f in [localPET,localSeg]:
@@ -100,9 +104,21 @@ def main(parameterFile):
                 os.remove(f)
 
         #build image path
-        remoteDir=fb.buildPathURL(project,[pars['imageDir'],\
-            r['patientCode'],r['visitCode']])
+        try:
+            remoteDir=fb.buildPathURL(project,[pars['imageDir'],\
+                r['patientCode'],r['visitCode']])
+        except TypeError:
+            print('[{}/{}]: Skipping - Failed to build remote dir'.\
+                    format(r[participantField],r['SequenceNum']))
+            continue
+
         print('{}: {}'.format(petField,r[petField]))
+
+        if r[petField]==None:
+            print('[{}/{}]: Skipping - Missing PET'.\
+                    format(r[participantField],r['SequenceNum']))
+            continue
+
         remotePET=remoteDir+'/'+r[petField]
         print('{}:{}'.format(remotePET,fb.entryExists(remotePET)))
 
@@ -112,7 +128,9 @@ def main(parameterFile):
 
         dsSeg=db.selectRows(project,segSchema,segQuery,[idFilter,visitFilter,vFilter])
         if len(dsSeg['rows'])!=1:
-            print('Failed to get segmentation for {}/{}'.format(r[participantField],segVersion))
+            print('[{}/{}] Skipping - missing segmentation, version [{}]'.\
+                    format(r[participantField],r['SequenceNum'],segVersion))
+            continue
 
         remoteSeg=remoteDir+'/'+dsSeg['rows'][0]['segmentation']
 

pythonScripts/preprocess.py

@@ -149,6 +149,20 @@ def main(parameterFile):
 
     i=0
     for row in ds["rows"]:
+
+        #check if imageSelector fields are set
+        orthancFields=[row[imageSelector[f]] for f in imageSelector]
+        #what does labkey return for empty fields - undefined/None
+        orthancFieldsOK=[f!=None for f in orthancFields]
+        #here I assume it returns a string with length 0
+        #orthancFieldsOK=[len(f)>0 for f in orthancFields]
+        if not all(orthancFieldsOK):
+            print('[{}/{}]: Missing orthanc fields:{}'.
+                    format(row[participantField],row['SequenceNum'],orthancFieldsOK))
+            continue
+        
+
+
         print("Starting row id:{} seq:{}".format(row[participantField],row['SequenceNum']))
         #interesting files are processedDir/studyName_CT_notCropped_2mmVoxel.nii
         #asn processedDir/studyName_PET_notCropped_2mmVoxel.nii

pythonScripts/runSegmentationnnUNet.py

@@ -175,7 +175,7 @@ def doSegmentation(parameterFile):
     dataset=pars['targetQuery']
     schema=pars['targetSchema']
     view=pars['viewName']
-
+    participantField=pars['participantField']
 
     tempBase=pars['tempBase']
     if not os.path.isdir(tempBase):
@@ -203,10 +203,16 @@ def doSegmentation(parameterFile):
     rows=ds['rows']
     for row in rows:
        
+        ctField=row[pars['images']['CT']['queryField']]
+        if ctField==None:
+            print('{}/{}: missing resampled CT'.format(row[participantField],row['SequenceNum']))
+            continue
+
 
         #build file name 
         sfx=pars['images']['segmentation']['suffix']
-        outpath=fb.buildPathURL(pars['project'],[pars['imageDir'],row['patientCode'],row['visitCode']])
+        outpath=fb.buildPathURL(pars['project'],\
+                [pars['imageDir'],row['patientCode'],row['visitCode']])
         outName=addVersion(\
                 getSegmImagePath(\
                     getStudyLabel(row,pars['participantField'])+sfx),\