irAEMM/pythonScripts/anonymizeSegmentations.py

import os
import json
import re
import subprocess
import nibabel
import shutil
import sys
import datetime

if len(sys.argv)<2:
    print("Usage {} version(v1 or similar)".format(sys.argv[0]))
    sys.exit(0)

#sourceDir=sys.argv[1]
ver=sys.argv[1]

shome=os.path.expanduser('~nixUser')
fhome=os.path.expanduser('~')
with open(os.path.join(fhome,".labkey","setup.json")) as f:
    setup=json.load(f)

sys.path.insert(0,setup["paths"]["labkeyInterface"])
import labkeyInterface
import labkeyDatabaseBrowser
import labkeyFileBrowser

#sys.path.insert(1,shome+'/software/src/IPNUMM/dicomUtils')
#import loadDicom

#onko
configOnko=os.path.join(fhome,'.labkey','onko-nix.json')
netOnko=labkeyInterface.labkeyInterface()
netOnko.init(configOnko)
dbOnko=labkeyDatabaseBrowser.labkeyDB(netOnko)
fbOnko=labkeyFileBrowser.labkeyFileBrowser(netOnko)


#merlin
configMerlin=os.path.join(fhome,'.labkey','merlin.json')
netMerlin=labkeyInterface.labkeyInterface()
netMerlin.init(configMerlin)
dbMerlin=labkeyDatabaseBrowser.labkeyDB(netMerlin)
fbMerlin=labkeyFileBrowser.labkeyFileBrowser(netMerlin)

projectOnko='iPNUMMretro/Study'
datasetOnko='Imaging1'

projectMerlin='iPNUMMretro/Study'
datasetMerlin='Imaging1'

tempBase=os.path.join(fhome,'temp')

#all images from database

#imageSelector={"CT":"CT","PET":"PETWB"};
imageResampledField={"Segm":"Segmentation"}
fileCode="Segm"

participantField='PatientId'
visitCode='visitCode'
#for prosepective
#participantField='ParticipantId'

#generate key from known imaging dates

def getImagingDates(db,project,dataset,participant,\
        participantField='PatientId'):
    idFilter={'variable':participantField,\
            'value':participant,
            'oper':'eq'}
    ds=db.selectRows(project,'study',dataset,[idFilter])
    return [datetime.datetime.strptime(row['studyDate'], '%Y/%m/%d %H:%M:%S')\
            for row in ds['rows']]

def getImagingDatesDictionary(db,project,dataset,\
        participantField='PatientId'):
    ds=db.selectRows(project,'study',dataset,[])
    participants=[row[participantField] for row in ds['rows']]
    #unique
    participants=list(set(participants))
    return {p:getImagingDates(db,project,dataset,p,participantField)\
            for p in participants}

def imagingDatesMatch(dateList1,dateList2):
    n=min(len(dateList1),len(dateList2))
    
    datesEqual=[abs(d2-d1)<datetime.timedelta(days=1) \
            for d1,d2 in zip(dateList1[:n],dateList2[:n])]

    return all(datesEqual)

def findMatch(dateList,dateDictionary):
    for p in dateDictionary:
        if imagingDatesMatch(dateList,dateDictionary[p]):
            return p
    return None


#use webdav to transfer file (even though it is localhost)


def getPatientLabel(row,participantField='PatientId'):
    return row[participantField].replace('/','_') 

def getVisitLabel(row):
    return 'VISIT_'+str(int(row['SequenceNum']))

def getStudyLabel(row,participantField='PatientId'):
    return getPatientLabel(row,participantField)+'-'+getVisitLabel(row)

def updateRow(db,project,dataset,row,imageResampledField,gzFileNames):
    for im in imageResampledField:
        row[imageResampledField[im]]=gzFileNames[im]
    db.modifyRows('update',project,'study',dataset,[row])
 
dOnko=getImagingDatesDictionary(dbOnko,projectOnko,datasetOnko,\
        participantField)
dMerlin=getImagingDatesDictionary(dbMerlin,projectMerlin,datasetMerlin,\
        participantField)

i=0
idMatch={d0:findMatch(dOnko[d0],dMerlin) for d0 in dOnko}

for id in idMatch:
    print("{} : {}".format(id,idMatch[id]))

dsOnko=dbOnko.selectRows(projectOnko,'study',datasetOnko,[])

im=list(imageResampledField.keys())[0]

for rowOnko in dsOnko["rows"]:
    
    
    segFileOnko=rowOnko[imageResampledField[im]]
    dirOnko=fbOnko.buildPathURL(projectOnko,\
            ['preprocessedImages',getPatientLabel(rowOnko,participantField),\
            getVisitLabel(rowOnko)])
    fileOnko=dirOnko+'/'+segFileOnko
    

    if not fbOnko.entryExists(fileOnko):
        print("Original file {} not available".format(fileOnko))
        continue
    
    
    
    
    idFilter={'variable':participantField,\
            'value':idMatch[rowOnko[participantField]],\
            'oper':'eq'}
    visitFilter={'variable':visitCode,
            'value':rowOnko[visitCode],
            'oper':'eq'}
    
    dsMerlin=dbMerlin.selectRows(projectMerlin,'study',datasetMerlin,\
            [idFilter,visitFilter])
    
    if len(dsMerlin['rows'])==0: 
        continue
    
    rowMerlin=dsMerlin['rows'][0]
    
    
    
    segFileMerlin=getStudyLabel(rowMerlin,participantField)+'_'+\
            im+'_'+ver+'.nii.gz'
    dirMerlin=fbMerlin.buildPathURL(projectMerlin,\
            ['preprocessedImages',getPatientLabel(rowMerlin,participantField),\
            getVisitLabel(rowMerlin)])
    fileMerlin=dirMerlin+'/'+segFileMerlin
    gzTargetFiles={im:segFileMerlin}

    if fbMerlin.entryExists(fileMerlin):
        print("Target file {} already uploaded".format(fileMerlin))
        updateRow(dbMerlin, projectMerlin,datasetMerlin,rowMerlin,\
                imageResampledField,gzTargetFiles)
        
        continue

    #upload file
    localFile=os.path.join(tempBase,segFileMerlin)
    print("Local {}".format(localFile))
    fbOnko.readFileToFile(fileOnko,localFile)
    fbMerlin.writeFileToFile(localFile,fileMerlin)
    print("Remote {}".format(fileMerlin))
    os.remove(localFile)

    #update row and let it know where the processed files are
    updateRow(dbMerlin, projectMerlin,datasetMerlin,rowMerlin,
            imageResampledField,gzTargetFiles)
   

    if i==-1:
        break
    i=i+1


print("Done")