irAEMM/pythonScripts/populateImagingFromTransferList.py

#date sorts studies from orthanc dataset into target study dataset

#takes transferQuery as the list of images that should be available on orthanc


import os
import json
import re
import sys
import datetime
import re

def parseDate(x):
    patterns=['%Y/%m/%d %H:%M:%S','%Y-%m-%d %H:%M:%S.%f']
    for p in patterns:
        try:
            return datetime.datetime.strptime(x,p)
        except ValueError:
            pass
    raise ValueError('parseDate could not find an appropriate pattern')
    return None

def main(parameterFile):
    fhome=os.path.expanduser('~')
    fsetup=os.path.join(fhome,'.labkey','setup.json')
    with open(fsetup,'r') as f:
        setup=json.load(f)

    sys.path.insert(0,setup['paths']['nixWrapper'])

    import nixWrapper

    nixWrapper.loadLibrary("labkeyInterface")

    import labkeyInterface
    import labkeyDatabaseBrowser
    import labkeyFileBrowser

    fconfig=os.path.join(fhome,'.labkey','network.json')

    net=labkeyInterface.labkeyInterface()
    net.init(fconfig)
    db=labkeyDatabaseBrowser.labkeyDB(net)
    fb=labkeyFileBrowser.labkeyFileBrowser(net)

    with open(parameterFile,'r') as f:
        pars=json.load(f)



    i=0
    #from orthancDatabase/Imaging dataset
    projectOrthanc=pars['Orthanc']['project']
    inputQuery=pars['Orthanc']['queryName']
    inputSchema=pars['Orthanc']['schemaName']
    inputParticipantField=pars['Orthanc']['participantField']

    #to target project dataset
    projectStudy=pars['Database']['project']
    #'iPNUMMretro/Study'
    #for prospective, set
    #projectStudy='IPNUMMprospektiva/Study'
    outputQuery=pars['Database']['queryName']
    outputSchema=pars['Database']['schemaName']
    #select patientId that are contained in the demographics dataset
    transferQuery=pars['Database']['transferQuery']
    dbParticipantField=pars['Database']['participantField']

    missingSchema=pars['Database']['missingImagesSchema']
    missingQuery=pars['Database']['missingImagesQuery']

    #delete all rows from missingQuery (should be rebuilt for every run)
    dsMissing=db.selectRows(projectStudy,missingSchema,missingQuery,[])
    db.modifyRows('delete',projectStudy,missingSchema,missingQuery,dsMissing['rows'])


    #make a list of images from transferQuery
    dsImage=db.selectRows(projectStudy,outputSchema,transferQuery,[])

    for im in dsImage['rows']:


        #for orthanc
        inputIdFilter={'variable':inputParticipantField,\
                'value':im[dbParticipantField],\
                'oper':'eq'}

        #for database
        idFilter={'variable':dbParticipantField,\
                'value':im[dbParticipantField],\
                'oper':'eq'}


        seqNum=im['imagingVisitId']
        seqFilter={'variable':'SequenceNum','value':str(seqNum),'oper':'eq'}

        
        #for speedup one should check if a match was already done in Database/queryName
        #ds1 are the matching outputs in target dataset
        ds1=db.selectRows(projectStudy,outputSchema,outputQuery,\
                [idFilter,seqFilter])

        if len(ds1['rows'])>1:
            #never happens (idFilter and seqFilter)
            print('ERROR: too many matches in {} for {}/{}'.\
                    format(outputQuery,im[dbParticipantField],seqNum))
            continue
        if len(ds1['rows'])>0:
            #just the one match, fine
            print('Entry for {}/{} already resolved'.\
                    format(im[dbParticipantField],seqNum))
            
            entry=ds1['rows'][0]
            try:
                if len(entry['PETWB_orthancId'])>0 and len(entry['CT_orthancId'])>0:
                    continue
            except:
                pass
            print('Debug mode for missing cases')

        #have to convert from datetime to %Y%m%d format
        #dateFilter={'variable':'imageDate','value':im['imageDate'],'oper':'eq'}

        dsOrthanc=db.selectRows(projectOrthanc,inputSchema,inputQuery,[inputIdFilter])

        #what if dsOrthanc['rows'] is empty?
        #this is part of QA and is reported in missingImages Schema/Query
    
        #convert dates
        sDates={r['orthancSeries']:parseDate(r['studyDate']) 
                for r in dsOrthanc['rows']}

        sDates={x:sDates[x].strftime('%Y%m%d') for x in sDates}

        #unique dates (for date mismatch)
        dates=list(set(sDates.values()))
        
        #select dates
        sDatesMatch={x:sDates[x] for x in sDates if sDates[x]==im['imageDate']}
        
        #select 
        rowsMatch=[r for r in dsOrthanc['rows'] if r['orthancSeries'] in list(sDatesMatch.keys())]

        #select CT matches
        rowsCT=[r for r in rowsMatch if r['seriesDescription'].find('CT WB')==0]
        #should not include fov
        rowsCT=[r for r in rowsCT if r['seriesDescription'].find('fov')<0]
        print('entry[{}/{}] rowsCT : {}'.format(im[dbParticipantField],seqNum,rowsCT))
        
        #should be one and one only
        rowsPET=[r for r in rowsMatch if r['seriesDescription']=='PET WB']
        print('entry[{}/{}] rowsPET: {}'.format(im[dbParticipantField],seqNum,rowsPET))
        
        #deal with erroneous outcomes (ie- no CT, more then 1 CT, no PET, more than 1 PET)
        if len(rowsPET)!=1 or len(rowsCT)!=1:
            mode='insert'
            #copy values en mass
            vals=[dbParticipantField,'imagingVisitId','imageDate']
            orow={v:im[v] for v in vals}
            orow['imageDateMismatch']=','.join(dates)
            
            #generate description of failure
            failDesc=''
            if len(rowsPET)==0:
                failDesc+='[MISSSING PET]'
            if len(rowsPET)>1:
                failDesc+='[MULTIPLE PET]'
            if len(rowsCT)==0:
                failDesc+='[MISSSING CT]'
            if len(rowsCT)>1:
                failDesc+='[MULTIPLE CT]'
            orow['failureDescription']=failDesc

            #generate fail entries
            db.modifyRows(mode,projectStudy,missingSchema,\
                    missingQuery,[orow])
            #don't break, but fill only for singular outcomes
         

        #figure out which row in output study to update
        filters=[]
        print('Participant {} Sequence number {}'.\
               format(im[dbParticipantField],str(seqNum)))
        #refresh the matching study list (after CT we have found PET)
        ds1=db.selectRows(projectStudy,outputSchema,outputQuery,\
                [idFilter,seqFilter])

        mode='update'
        outRow={}
        if len(ds1['rows'])==0:
            mode='insert'
        
            outRow[dbParticipantField]=im[dbParticipantField]
            outRow['SequenceNum']=seqNum
        
        else:
            #never happens if we check for sd1 before matches are found
            outRow=ds1['rows'][0]
        
        if len(rowsPET)==1:    
            outRow['PETWB_orthancId']=rowsPET[0]['orthancSeries']
            outRow['studyDate']=rowsPET[0]['studyDate']
        if len(rowsCT)==1:
            outRow['CT_orthancId']=rowsCT[0]['orthancSeries']
            outRow['studyDate']=rowsCT[0]['studyDate']

        outRow['imagingVisitId']=im['imagingVisitId']
        outRow['visitCode']='VISIT_'+str(im['imagingVisitId'])

        modifyStatus=db.modifyRows(mode,projectStudy,outputSchema,\
                outputQuery,[outRow])
        print('{}'.format(modifyStatus))
        
    print("Done")

if __name__=='__main__':
    main(sys.argv[1])