irAEMM/pythonScripts/populateImagingFromTransferList.py

#date sorts studies from orthanc dataset into target study dataset

#takes transferQuery as the list of images that should be available on orthanc


import os
import json
import re
import sys
import datetime
import re

def main(parameterFile):
    fhome=os.path.expanduser('~')
    fsetup=os.path.join(fhome,'.labkey','setup.json')
    with open(fsetup,'r') as f:
        setup=json.load(f)

    sys.path.insert(0,setup['paths']['nixWrapper'])

    import nixWrapper

    nixWrapper.loadLibrary("labkeyInterface")

    import labkeyInterface
    import labkeyDatabaseBrowser
    import labkeyFileBrowser

    fconfig=os.path.join(fhome,'.labkey','network.json')

    net=labkeyInterface.labkeyInterface()
    net.init(fconfig)
    db=labkeyDatabaseBrowser.labkeyDB(net)
    fb=labkeyFileBrowser.labkeyFileBrowser(net)

    with open(parameterFile,'r') as f:
        pars=json.load(f)



    i=0
    #from orthancDatabase/Imaging dataset
    projectOrthanc=pars['Orthanc']['project']
    inputQuery=pars['Orthanc']['queryName']
    inputSchema=pars['Orthanc']['schemaName']
    inputParticipantField=pars['Orthanc']['participantField']

    #to target project dataset
    projectStudy=pars['Database']['project']
    #'iPNUMMretro/Study'
    #for prospective, set
    #projectStudy='IPNUMMprospektiva/Study'
    outputQuery=pars['Database']['queryName']
    outputSchema=pars['Database']['schemaName']
    #select patientId that are contained in the demographics dataset
    transferQuery=pars['Database']['transferQuery']
    dbParticipantField=pars['Database']['participantField']

    missingSchema=pars['Database']['missingImagesSchema']
    missingQuery=pars['Database']['missingImagesQuery']

    #make a list of images from transferQuery
    dsImage=db.selectRows(projectStudy,outputSchema,transferQuery,[])

    for im in dsImage['rows']:


        #for orthanc
        inputIdFilter={'variable':inputParticipantField,\
                'value':im[dbParticipantField],\
                'oper':'eq'}

        #for database
        idFilter={'variable':dbParticipantField,\
                'value':im[dbParticipantField],\
                'oper':'eq'}


        seqNum=im['imagingVisitId']
        seqFilter={'variable':'SequenceNum','value':str(seqNum),'oper':'eq'}

        
        #for speedup one should check if a match was already done in Database/queryName
        #ds1 are the matching outputs in target dataset
        ds1=db.selectRows(projectStudy,outputSchema,outputQuery,\
                [idFilter,seqFilter])

        if len(ds1['rows'])>1:
            #never happens (idFilter and seqFilter)
            print('ERROR: too many matches in {} for {}/{}'.\
                    format(outputQuery,im[dbParticipantField],seqNum))
            continue
        if len(ds1['rows'])>0:
            #just the one match, fine
            print('Entry for {}/{} already resolved'.\
                    format(im[dbParticipantField],seqNum))
            continue

        #have to convert from datetime to %Y%m%d format
        #dateFilter={'variable':'imageDate','value':im['imageDate'],'oper':'eq'}

        dsOrthanc=db.selectRows(projectOrthanc,inputSchema,inputQuery,[inputIdFilter])
        #what if dsOrthanc['rows'] is empty?
        #this is part of QA and is reported in missingImages Schema/Query

        uploadStatus="FAILED"
        imageDateMismatch=[]
        for im1 in dsOrthanc['rows']:

            date=datetime.datetime.strptime(im1['studyDate'],'%Y/%m/%d %H:%M:%S') 
            #convert date to %Y%m%d notation
            dateYMD=date.strftime('%Y%m%d')
            if dateYMD!=im['imageDate']:
                print('Rejecting mismatch: {}/{}'.format(dateYMD,im['imageDate']))
                imageDateMismatch.append(dateYMD)
                continue
        
            outvar='NONE'
            sd=im1['seriesDescription']
            if sd=='PET WB':
                outvar='PETWB_orthancId'
                print('Found PET: {}'.format(im1['orthancSeries']))
            if sd.find('CT WB')==0:
                if sd.find('fov')<0:
                    outvar='CT_orthancId'
                    print('Found CT: {}'.format(im1['orthancSeries']))

            #skip irrelevant series
            if outvar=='NONE':
                continue

            #figure out which row in output study to update
            filters=[]
            print('Participant {} Sequence number {}'.\
                    format(im[dbParticipantField],str(seqNum)))
    

            mode='update'
            outRow={}
            if len(ds1['rows'])==0:
                mode='insert'
        
                outRow[dbParticipantField]=im[dbParticipantField]
                outRow['SequenceNum']=seqNum
                outRow['dicomStudy']=im1['dicomStudy']
        
            else:
                #never happens if we check for sd1 before matches are found
                outRow=ds1['rows'][0]
        
            
            outRow[outvar]=im1['orthancSeries']
            outRow['studyDate']=im1['studyDate']
            outRow['imagingVisitId']=im['imagingVisitId']
            outRow['visitCode']='VISIT_'+str(im['imagingVisitId'])

            modifyStatus=db.modifyRows(mode,projectStudy,outputSchema,\
                    outputQuery,[outRow])
            print('{}'.format(modifyStatus))
            uploadStatus="LOADED"
        
        if uploadStatus=="FAILED":
            #standard spiel - find if already present; if so, skip, if not, add
            imFilter={'variable':'imagingVisitId',
                    'value':'{}'.format(im['imagingVisitId']),
                    'oper':'eq'}
            dsMissing=db.selectRows(projectStudy,missingSchema,\
                    missingQuery,[idFilter,imFilter])

            #already recorded
            imageDateMismatch=list(set(imageDateMismatch))
            vals=[dbParticipantField,'imagingVisitId','imageDate']
            mode='insert'
            if len(dsMissing['rows'])>0:
                mode='update'
                orow=dsMissing['rows'][0]
            else:
                orow={v:im[v] for v in vals}
            orow['imageDateMismatch']=','.join(imageDateMismatch)
            orow['failureDescription']='MISSING DICOM'
            db.modifyRows(mode,projectStudy,missingSchema,\
                    missingQuery,[orow])
            
       
    print("Done")

if __name__=='__main__':
    main(sys.argv[1])