{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "3749879d-7d4f-443d-a8a1-0b2dc7b379ae", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "loadLibrary\n", "remoteSourcesURL https://git0.fmf.uni-lj.si/studen/nixSuite/raw/master/remoteResources/resources.json\n", "{'labkeyInterface': {'url': 'https://git0.fmf.uni-lj.si/studen/labkeyInterface/archive/master.zip', 'branch': 'master', 'modules': []}, 'irAEMM': {'url': 'https://git0.fmf.uni-lj.si/studen/iraemm/archive/master.zip', 'branch': 'master', 'modules': ['iraemmBrowser']}, 'SlicerLabkeyExtension': {'url': 'https://git0.fmf.uni-lj.si/studen/SlicerLabkeyExtension/archive/SlicerExtensionIndex.zip', 'branch': 'SlicerExtensionIndex', 'modules': ['labkeyBrowser']}, 'limfomiPET': {'url': 'https://git0.fmf.uni-lj.si/studen/limfomiPET/archive/master.zip', 'branch': 'master', 'modules': ['imageBrowser', 'segmentationBrowser']}, 'parseConfig': {'url': 'https://git0.fmf.uni-lj.si/studen/parseConfig/archive/master.zip', 'branch': 'master', 'modules': []}, 'orthancInterface': {'url': 'https://git0.fmf.uni-lj.si/studen/orthancInterface/archive/master.zip', 'branch': 'master', 'modules': []}}\n", "{'url': 'https://git0.fmf.uni-lj.si/studen/labkeyInterface/archive/master.zip', 'branch': 'master', 'modules': []}\n", "File C:\\Users\\studen\\temp\\labkeyInterface.zip: True\n", "loadLibrary\n", "remoteSourcesURL https://git0.fmf.uni-lj.si/studen/nixSuite/raw/master/remoteResources/resources.json\n", "{'labkeyInterface': {'url': 'https://git0.fmf.uni-lj.si/studen/labkeyInterface/archive/master.zip', 'branch': 'master', 'modules': []}, 'irAEMM': {'url': 'https://git0.fmf.uni-lj.si/studen/iraemm/archive/master.zip', 'branch': 'master', 'modules': ['iraemmBrowser']}, 'SlicerLabkeyExtension': {'url': 'https://git0.fmf.uni-lj.si/studen/SlicerLabkeyExtension/archive/SlicerExtensionIndex.zip', 'branch': 'SlicerExtensionIndex', 'modules': ['labkeyBrowser']}, 'limfomiPET': {'url': 'https://git0.fmf.uni-lj.si/studen/limfomiPET/archive/master.zip', 'branch': 'master', 'modules': ['imageBrowser', 'segmentationBrowser']}, 'parseConfig': {'url': 'https://git0.fmf.uni-lj.si/studen/parseConfig/archive/master.zip', 'branch': 'master', 'modules': []}, 'orthancInterface': {'url': 'https://git0.fmf.uni-lj.si/studen/orthancInterface/archive/master.zip', 'branch': 'master', 'modules': []}}\n", "{'url': 'https://git0.fmf.uni-lj.si/studen/orthancInterface/archive/master.zip', 'branch': 'master', 'modules': []}\n", "File C:\\Users\\studen\\temp\\orthancInterface.zip: True\n" ] } ], "source": [ "#load required libraries\n", "import sys\n", "import os\n", "import chardet\n", "import json\n", "import re\n", "import datetime\n", "\n", "#you should get nixSuite via git clone https://git0.fmf.uni-lj.si/studen/nixSuite.git\n", "#if you don't put it to $HOME/software/src/, you should update the path\n", "nixSuite=os.path.join(os.path.expanduser('~'),'software','src','nixSuite')\n", "sys.path.append(os.path.join(nixSuite,'wrapper'))\n", "import nixWrapper\n", "nixWrapper.loadLibrary('labkeyInterface')\n", "import labkeyInterface\n", "import labkeyDatabaseBrowser\n", "import labkeyFileBrowser\n", "nixWrapper.loadLibrary('orthancInterface')\n", "import orthancInterface\n", "import orthancDatabaseBrowser\n", "\n", "def connectDB(server):\n", " #check connectivity. This checks the configuration in $HOME/.labkey/network.json, \n", " #where paths to certificates are stored\n", " net=labkeyInterface.labkeyInterface()\n", " fconfig=os.path.join(os.path.expanduser('~'),'.labkey','{}.json'.format(server))\n", " net.init(fconfig)\n", " #this reports the certificate used\n", " try:\n", " print('Using: {}'.format(net.connectionConfig['SSL']['user']))\n", " except KeyError:\n", " pass\n", " #This gets a deafult CSRF code; It should report user name plus a long string of random hex numbers\n", " net.getCSRF()\n", " db=labkeyDatabaseBrowser.labkeyDB(net)\n", " fb=labkeyFileBrowser.labkeyFileBrowser(net)\n", " return db,fb\n", "\n", "def modifyRowsWithCheck(db,project,schema,query,entry,testFields):\n", " mode='insert'\n", " filters=[{'variable':x,'value':'{}'.format(entry[x]),'oper':'eq'} for x in testFields]\n", " ds=db.selectRows(project,schema,query,filters)\n", " finalEntry={}\n", " if len(ds['rows'])==1:\n", " finalEntry=ds['rows'][0]\n", " mode='update'\n", " for q in entry:\n", " finalEntry[q]=entry[q];\n", " print(db.modifyRows(mode,project,schema,query,[finalEntry]))\n", "\n", "def connectOrthanc(server):\n", " net=orthancInterface.orthancInterface()\n", " fconfig=os.path.join(os.path.expanduser('~'),'.labkey','{}.json'.format(server))\n", " net.init(fconfig)\n", " db=orthancDatabaseBrowser.orthancDB(net)\n", " return db" ] }, { "cell_type": "code", "execution_count": 18, "id": "7c8f8fec-bae9-4207-9a60-1b1644939a58", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "User: andrej studen CSRF: 48edbd6b5597865fd8127dbca709dd85\n", "280/424\n", "290/424\n", "300/424\n", "310/424\n", "320/424\n", "330/424\n", "340/424\n", "350/424\n", "360/424\n", "370/424\n", "380/424\n", "390/424\n", "400/424\n", "410/424\n", "420/424\n" ] } ], "source": [ "#check if all entries from a list appear in the matching dataset\n", "db=connectDB('onko-nix')[0]\n", "project='limfomiPET/Study'\n", "query='imageTransferReport'\n", "\n", "dl=db.selectRows(project,'lists',query,[])\n", "\n", "startWith=270\n", "rows=dl['rows'][startWith:]\n", "filterArray=['participantCode','imagingVisitId']\n", "n=len(rows)+startWith\n", "i=startWith\n", "crfs=[]\n", "for r in rows:\n", " qfilter=[{'variable':x,'value':'{}'.format(r[x]),'oper':'eq'} for x in filterArray]\n", " ds=db.selectRows(project,'study',query,qfilter)\n", " d=[r[x] for x in filterArray]\n", " if len(ds['rows'])!=1:\n", " print('Got {} rows for {}/{}'.format(len(ds['rows']),d[0],d[1]))\n", " for q in ds['rows']:\n", " print(q)\n", " redundantRow=ds['rows'][-1]\n", " redundantCrf=redundantRow['crfRef']\n", " crfs.append(redundantCrf)\n", " i+=1\n", " if i%10 == 0:\n", " print('{}/{}'.format(i,n))\n", " #else:\n", " # print('Found {}/{}'.format(d[0],d[1]))\n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "2d09b6d0-5ce9-49dc-8690-245f5ea97070", "metadata": {}, "outputs": [], "source": [ "#deal with duplicates\n", "query='imageTransferReport'\n", "crfs=list(set(crfs))\n", "print(len(crfs))\n", "crfEntry={'schema':'lists','query':'crfEntry','ref':'entryId'}\n", "q1={'schema':'lists','query':query,'ref':'crfRef'}\n", "q2={'schema':'study','query':query,'ref':'crfRef'}\n", "queries=[crfEntry,q1,q2]\n", "\n", "for c in crfs:\n", " for q in queries:\n", " filter={'variable':q['ref'],'value':c,'oper':'eq'}\n", " dsX=db.selectRows(project,q['schema'],q['query'],[filter])\n", " print('{}/{} {} {}'.format(q['schema'],q['query'],c,len(dsX['rows'])))\n", " db.modifyRows('delete',project,q['schema'],q['query'],dsX['rows'])" ] }, { "cell_type": "code", "execution_count": null, "id": "e617b59b-6c94-429a-b9ca-082f8e91e33b", "metadata": {}, "outputs": [], "source": [ "#find mismatched Id labels (spaces etc)\n", "def deleteWrongIDs(query,varList={}):\n", " db=connectDB('onko-nix')[0]\n", " project='limfomiPET/Study'\n", " \n", " qFilter=[{'variable':x,'value':'{}'.format(varList[x]),'oper':'eq'} for x in varList]\n", " ds=db.selectRows(project,'study',query,qFilter)\n", "\n", " pattern=r'^ ([^ ]*) $'\n", " rows=[r for r in ds['rows'] if re.match(pattern,r['ParticipantId'])]\n", "\n", "\n", " for r in rows:\n", " id=r['ParticipantId']\n", " print('[{}]'.format(id))\n", " db.modifyRows('delete',project,'study',query,[r])\n", "\n", "def checkIDs(query,varList={}):\n", " db=connectDB('onko-nix')[0]\n", " project='limfomiPET/Study'\n", " refQuery='Imaging1'\n", " ds=db.selectRows(project,'study',refQuery,[])\n", " ids=list(set([r['ParticipantId'] for r in ds['rows']]))\n", " i=0\n", " for id in ids:\n", " qFilter=[]\n", " qFilter.append({'variable':'ParticipantId','value':'{}'.format(id),'oper':'eq'})\n", " qFilter.extend([{'variable':x,'value':'{}'.format(varList[x]),'oper':'eq'} for x in varList])\n", " ds=db.selectRows(project,'study',query,qFilter)\n", " if len(ds['rows'])>0:\n", " continue\n", " i+=1\n", " print('[{}] Missing [{}]'.format(i,id))\n", " \n", " print('Total: {}'.format(i)) \n", "\n", "#query='cancerData'\n", "#query='cancerTreatment'\n", "query='events'\n", "#varList={'eventType':1}\n", "#varList={'eventType':2}\n", "varList={'eventType':3}\n", "deleteWrongIDs(query,varList)\n", "checkIDs(query,varList)\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "7b36a7a5-792f-448e-becb-f70238bc715b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "User: andrej studen CSRF: 1cb6c1759ca692ab6cd596633ab86afd\n", "2018-06-20 00:00:00 4763/18 [157.88728125, 158.75828124999998, 849.0]\n", "2019-01-07 00:00:00 4763/18 [157.51428125, 157.47828125, 975.0]\n", "2019-11-15 00:00:00 4763/18 [171.24303125, 173.97103125, 975.0]\n" ] } ], "source": [ "#check imagePositionPatient matches in CT/PET\n", "\n", "def getPos(orthanc,orthancSeriesId):\n", " instance=orthanc.getSeriesData(orthancSeriesId)['Instances'][0]\n", " return [float(x) for x in orthanc.getDicomField(instance,'0020-0032').split('\\\\')]\n", "\n", " \n", " \n", "def checkPos():\n", " db=connectDB('onko-nix')[0]\n", " orthanc=connectOrthanc('onko-nix')\n", " project='limfomiPET/Study'\n", " query='Imaging1'\n", " ds=db.selectRows(project,'study',query,[])\n", " rows=[r for r in ds['rows'] if r['ParticipantId']=='4763/18']\n", " diff={}\n", " for r in rows:\n", " date=datetime.datetime.strptime(r['studyDate'],'%Y/%m/%d %H:%M:%S')\n", " #print(date)\n", " ct=r['CT_orthancId']\n", " pet=r['PETWB_orthancId']\n", " posCT=getPos(orthanc,ct)\n", " posPET=getPos(orthanc,pet)\n", " d=[a-b for (a,b) in zip(posCT,posPET)]\n", " #print('{} {} {}'.format(posCT,posPET,d))\n", " diff[date]={'id':r['ParticipantId'],'diff':d}\n", " s=sorted(diff)\n", " for x in s:\n", " print('{} {} {}'.format(x,diff[x]['id'],diff[x]['diff']))\n", "checkPos()\n", " \n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "d32299fb-05a3-4ae0-85f9-90d2b54472de", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.0" } }, "nbformat": 4, "nbformat_minor": 5 }