limfomiPET/pythonScripts/anonymize.py

import os
import json
import re
import sys
import numpy
import shutil

#nothing gets done if you do import

def getPatientLabel(row,participantField='PatientId'):
    return row[participantField].replace('/','_') 

def getVisitLabel(row):
    return 'VISIT_'+str(int(row['SequenceNum']))

def getStudyLabel(row,participantField='PatientId'):
    return getPatientLabel(row,participantField)+'-'+getVisitLabel(row)

def main(parameterFile):
    fhome=os.path.expanduser('~')
    with open(os.path.join(fhome,".labkey","setup.json")) as f:
        setup=json.load(f)

    sys.path.insert(0,setup["paths"]["nixWrapper"])
    import nixWrapper

    nixWrapper.loadLibrary("labkeyInterface")

    import labkeyInterface
    import labkeyDatabaseBrowser
    import labkeyFileBrowser

    fconfig=os.path.join(fhome,'.labkey','onko-nix.json')

    net=labkeyInterface.labkeyInterface()
    net.init(fconfig)
    db=labkeyDatabaseBrowser.labkeyDB(net)
    fb=labkeyFileBrowser.labkeyFileBrowser(net)

    with open(parameterFile) as f:
        pars=json.load(f)

    hi=0
    project=pars['Database']['project']
    dataset=pars['Database']['queryName']
    schema=pars['Database']['schemaName']

    #tempBase=os.path.join('/','data','nixUser','RIS')
    tempBase=os.path.join(fhome,'temp','RIS')
    if not os.path.isdir(tempBase):
        os.mkdir(tempBase)


    participantField=pars['Database']['participantField']
    segmentation=pars['Database']['segmentationQuery']

    #all images from database
    visitFilter={'variable':'visitCode','value':'VISIT_2','oper':'eq'}
    iodineFilter={'variable':'iodineContrast','value':'1','oper':'neq'}
    #for VISIT_1, also apply iodineFilter
    #qFilter=[visitFilter,iodineFilter]
    #for VISIT_2, iodineFilter has no meaning (shuld be false or blank, but or is hard to do)
    qFilter=[visitFilter]

    #shift generated patient names
    offset=100

    ds=db.selectRows(project,schema,dataset,qFilter)
    #imageSelector={"CT":"CT_orthancId","PET":"PETWB_orthancId"}
    #output
    imageResampledField={"CT":"ctResampled","PET":"petResampled"}
    #,"patientmask":"ROImask"}

    #use webdav to transfer file (even though it is localhost)

    i=0

    n=len(ds['rows'])
    keys=[r[participantField] for r in ds['rows']]
    perm=numpy.random.permutation(n)
    perm+=offset
    pseudo={keys[i]:perm[i] for i in range(n)}
    
    for row in ds["rows"]:
        print("Starting row id:{} seq:{}".format(row[participantField],row['SequenceNum']))
        #interesting files are processedDir/studyName_CT_notCropped_2mmVoxel.nii
        #asn processedDir/studyName_PET_notCropped_2mmVoxel.nii

        idFilter={'variable':participantField,'value':row[participantField],'oper':'eq'}
        segFilter={'variable':'SequenceNum','value':'{}'.format(row['SequenceNum']),'oper':'eq'}
        #adoma
        userFilter={'variable':'User','value':'1037','oper':'eq'}
        ds=db.selectRows(project,schema,segmentation,[idFilter,segFilter,userFilter])
        nS=len(ds['rows'])
        if nS==0:
            print('No segmentation found')
            continue
        if nS>1:
            print('Multiple segmentations found')
            continue
        maskFile={'mask':'/'.join(['Segmentations',ds['rows'][0]['latestFile']])}
    
        #build/check remote directory structure
        remoteDir=fb.buildPathURL(project,['preprocessedImages',\
            getPatientLabel(row,participantField),getVisitLabel(row)])
        
        remoteFiles={x:row[imageResampledField[x]] for x in imageResampledField}
        remoteFiles.update(maskFile)
        remoteFiles={x:'/'.join([remoteDir,remoteFiles[x]]) for x in remoteFiles}
        for f in remoteFiles.values():
            print("[{}]: [{}]".format(f,fb.entryExists(f)))
        patientALabel='patient{:03d}'.format(pseudo[row[participantField]])
        localDir=os.path.join(tempBase,patientALabel)
        if not os.path.isdir(localDir):
            os.mkdir(localDir)
        fileNames={x:'{}.nii.gz'.format(x) for x in remoteFiles}
        fileNames['mask']=fileNames['mask'].replace('nii.gz','nrrd')
        localFiles={x:os.path.join(localDir,fileNames[x]) for x in fileNames}
        
        remoteFilesPresent={x:fb.entryExists(remoteFiles[x]) for x in remoteFiles}
        if not all(remoteFilesPresent):
            print('Missing remote files')
            continue
    

        _=[fb.readFileToFile(remoteFiles[x],localFiles[x]) for x in localFiles]

        remoteADir=fb.buildPathURL(project,['anonymized',patientALabel])
        remoteAFiles={x:'/'.join([remoteADir,fileNames[x]]) for x in fileNames}
        _=[fb.writeFileToFile(localFiles[x],remoteAFiles[x]) for x in remoteAFiles]

        shutil.rmtree(localDir)

        if i==-1:
            break
        i=i+1

    print("Done")


if __name__ == '__main__':
    main(sys.argv[1])