Rewritten portion of scanOrthancQuick to deal better with large Orthanc and Labkey databases

pythonScripts/scanOrthancQuick.py

@@ -16,6 +16,26 @@ import json
 import re
 import sys
 
+def selectPatient(orthancId,globalId,pars):
+    #pars is database portion of pars
+    if not pars.get('filter',None):
+        return True
+    qf=pars['filter']
+    #AND of possible filters
+    for f in qf:
+        #does not start with
+        if f=='does_not_start_with':
+            if globalId.startswith(qf[f]):
+                return False
+    return True
+
+def adjustId(fullId):
+    idLength=len(fullId)
+    if idLength<33:
+        return fullId,fullId
+    shortId=fullId.replace('-','')
+    return fullId,shortId
+
 def main(parameterFile):
 
     fhome=os.path.expanduser('~')
@@ -67,40 +87,74 @@ def main(parameterFile):
     participantField=opars['participantField']
 
     patients=odb.getPatients()
+    
 
+    print('Got: {} patients in Orthanc.'.format(len(patients)))
 
     #equivalent for labkey side?
     dsDemo=db.selectRows(project,opars['schemaName'],\
             opars['demographicsQuery'],[])
     dsPatients=[row['OrthancId'] for row in dsDemo['rows']]
+    print('Got {} patients in LabKey'.format(len(dsPatients)))
+
     pMissing=[p for p in patients if p not in dsPatients]
     print('Missing : {}'.format(len(pMissing)))
 
     #we need details for all patients (some might have just a study uploaded
-    pdata={p:odb.getPatientData(p) for p in patients}
-    
+    #this takes a while since demo table is not updated.
+    #pdata={p:odb.getPatientData(p) for p in patients}
+    #print('Got data for missing patients') 
+    #return
+
     #update patient data for missing patients
     prows=[]
+    i=0
     for p in pMissing:
-        dicom=pdata[p]['MainDicomTags']
+
+        #dicom=pdata[p]['MainDicomTags']
+        try:
+            dicom=odb.getPatientData(p)['MainDicomTags']
+        except TypeError:
+            continue
         row={}
-        row[participantField]=dicom['PatientID']
+        fullId,shortId=adjustId(dicom['PatientID'])
+        row[participantField]=shortId
+        idLength=len(shortId)
         row['birthDate']=dicom['PatientBirthDate']
         row['PatientName']=dicom['PatientName']
         row['OrthancId']=p
+        row['fullId']=fullId
         prows.append(row)
-
-    db.modifyRows('insert',project,opars['schemaName'],\
-                    opars['demographicsQuery'],prows)
+        oid=row[participantField]
+        print(f'Adding [{oid}]: {p}')
+        resp=db.modifyRows('insert',project,opars['schemaName'],
+                opars['demographicsQuery'],[row])
+        try:
+            print('[{}] Error: {}'.format(idLength,resp['exception']))
+        except KeyError:
+            pass
+        #if i==0:
+        #    return
+    #db.modifyRows('insert',project,opars['schemaName'],\
+                  #  opars['demographicsQuery'],prows)
 
     n=len(patients)
     i=0
+    prows.extend(dsDemo['rows'])
+    patientMap={r['OrthancId']:r[participantField] for r in prows}
+    
+    patients=[p for p in patients if selectPatient(p,patientMap[p],pars['Database'])]
+    print('After filter: {}/{}'.format(len(patients),n))
+    n=len(patients)
+
     #download the full images dataset (~1000 rows)
     ds=db.selectRows(project,opars['schemaName'],opars['queryName'],[])
+    print('Loaded image dataset ({} rows)'.format(len(ds['rows'])))
     for p in patients:
-        dicom=pdata[p]['MainDicomTags']
-        patientId=dicom['PatientID']
-
+        pdata=odb.getPatientData(p)
+        #dicom=pdata[p]['MainDicomTags']
+        #patientId=dicom['PatientID']
+        patientId=patientMap[p]
         #get all studies for pateint
         #qfilter={'variable':participantField,'value':patientId,'oper':'eq'}
         dsStudies=[row['orthancStudy'] for row in ds['rows'] if row[participantField]==patientId]
@@ -109,7 +163,7 @@ def main(parameterFile):
         
         #unique
         dsStudies=list(set(dsStudies))
-        studies=pdata[p]['Studies']
+        studies=pdata['Studies']
         pHex=p.split('-')
 
         print("[{:>3d}/{:>3d}] ID: {:>10s} [{}] Studies: {:>2}/{:>2} Entries in DB: {:>4d}".format(i,n,patientId,pHex[-1],len(dsStudies),len(studies),seqNum))