import qt import urllib3 import xml.etree.ElementTree as ET import re import slicer import shutil import distutils import os import json import chardet import io #mimics labkeyInterface class slicerNetwork(slicer.ScriptedLoadableModule.ScriptedLoadableModule): def __init__(self, parent): slicer.ScriptedLoadableModule.ScriptedLoadableModule.__init__(self,parent) self.parent.title="slicerNetwork" pass class DummyResponse: pass class labkeyURIHandler(slicer.vtkURIHandler): def __init__(self): slicer.vtkURIHandler.__init__(self) self.className="labkeyURIHandler" slicer.mrmlScene.AddURIHandler(self) fhome=os.path.expanduser('~') self.localCacheDirectory=os.path.join(fhome,"labkeyCache") self.configDir=os.path.join(fhome,".labkey") #try initializing network from default config file, if found self.initFromConfig() def CanHandleURI(self,uri): print("labkeyURIHandler::CanHandleURI({0})".format(uri)) if uri.find('labkey://')==0: return 1 return 0 def GetClassName(self): return self.className def GetHostName(self): self.hostname=self.connectionConfig['host'] print("Host name:{}".format(self.hostname)) return self.hostname def SetHostName(self,host): try: self.connectionConfig['host']=host except AttributeError: self.connectionConfig={} self.connectionConfig['host']=host self.hostname=host def SetLocalCahceDirectory(self,dir): self.localCacheDirectory=dir def GetLocalCacheDirectory(self): return self.localCacheDirectory def GetLabkeyUrl(self): return self.GetHostName()+"/labkey" def GetLabkeyWebdavUrl(self): return self.GetLabkeyUrl()+"/_webdav" #configuration part def configureSSL(self,cert=None,key=None,pwd=None,cacert=None): print("configure SSL: {}".format(cert)) if not cert==None: self.http = urllib3.PoolManager(\ cert_file=cert,\ cert_reqs='CERT_REQUIRED',\ key_file=key,\ ca_certs=cacert) else: self.http = urllib3.PoolManager(\ cert_reqs='CERT_REQUIRED',\ ca_certs=cacert) def initRemote(self): #assume self.connectionConfig is set self.http=urllib3.PoolManager() try: config=self.connectionConfig except AttributeError: print("ConnectionConfig not initialized") return if 'SSL' in self.connectionConfig: print("Setting up SSL") try: cert=self.connectionConfig['SSL']['user'] except KeyError: print("No user cert supplied") return try: key=self.connectionConfig['SSL']['key'] except KeyError: print("No user key supplied") return try: ca=self.connectionConfig['SSL']['ca'] except KeyError: print("No CA cert supplied") return self.configureSSL(cert,key,None,ca) def initFromConfig(self): print("slicerNetwork: initFromConfig") path=os.path.join(self.configDir,"Remote.json") try: self.parseConfig(path) except OSError: return self.initRemote() def parseConfig(self,fname,parent=None): print("slicerNetwork: parseConfig") try: with open(fname) as f: self.connectionConfig=json.load(f) except OSError as e: print("Confgiuration error: OS error({0}): {1}".format(e.errno, e.strerror)) raise except AttributeError: print("Troubles parsing json at {}".format(fname)) if not parent==None: parent.userCertButton.setText(self.connectionConfig['SSL']['user']) parent.caCertButton.setText(self.connectionConfig['SSL']['ca']) parent.privateKeyButton.setText(self.connectionConfig['SSL']['key']) parent.pwd=self.connectionConfig['SSL']['keyPwd'] self.hostname=self.connectionConfig['host'] self.auth_name=self.connectionConfig['labkey']['user'] self.auth_pass=self.connectionConfig['labkey']['password'] if not parent==None: parent.serverURL.setText(self.connectionConfig['host']) parent.authName.setText(self.connectionConfig['labkey']['user']) parent.authPass.setText(self.connectionConfig['labkey']['password']) #path convention #localPath is a path on local filesystem. It means it has to adhere to OS #policy, especially in separators #relativePath is just the portion of the path that is equal both locally #and remotely. In relativePath, separator is according to http convention, #which equals separator in osx and *nix #labkeyPath or remotePath is the full URL to the resource, #including http-like header and all intermediate portions #the following functions convert between different notations #was GetLocalPath def GetLocalPathFromRelativePath(self,relativePath): debug=False relativePath=re.sub('labkey://','',relativePath) sp=os.sep.encode('unicode_escape').decode('utf-8') if debug: print("Substituting / with {0} in {1}".format(sp,relativePath)) relativePath=re.sub('/',sp,relativePath) return os.path.join(self.GetLocalCacheDirectory(),relativePath) #was GetRemotePath def GetLabkeyPathFromRelativePath(self,source): return self.GetLabkeyWebdavUrl()+"/"+source #was GetLabkeyPathFromLocalPath def GetRelativePathFromLocalPath(self,f): debug=False #report it with URL separator, forward-slash if f.find(self.localCacheDirectory)<-1: print("Localpath misformation. Exiting") return "NULL" f0=re.sub('\\\\','\\\\\\\\',self.localCacheDirectory) relativePath=re.sub(re.compile(f0),'',f) if debug: print("[SEP] Relative path {}".format(relativePath)) #leaves /ContextPath/%40files/subdirectory_list/files #remove first separator sp=os.path.sep f1=re.sub('\\\\','\\\\\\\\',sp) if relativePath[0]==sp: relativePath=relativePath[1:len(relativePath)] if debug: print("[SLASH] Relative path {}".format(relativePath)) return re.sub(f1,'/',relativePath) #was GetLabkeyPathFromRemotePath def GetRelativePathFromLabkeyPath(self,f): #used to query labkey stuff, so URL separator is used #in old conventions, labkey://was used to tag a labkeyPath f=re.sub('labkey://','',f) f=re.sub(self.GetLabkeyWebdavUrl(),'',f) if f[0]=='/': f=f[1:len(f)] return f; def getBasicAuth(self): user=self.connectionConfig['labkey']['user'] pwd=self.connectionConfig['labkey']['password'] #debug return user+":"+pwd #standard HTTP def get(self,url,binary=False): debug=True if debug: print("GET: {0}".format(url)) print("as {0}".format(self.connectionConfig['labkey']['user'])) headers=urllib3.util.make_headers(basic_auth=self.getBasicAuth()) try: if not binary: return self.http.request('GET',url,headers=headers) else: return self.http.request('GET',url,headers=headers,preload_content=False) #f contains json as a return value #f contains json as a return value except urllib3.exceptions.HTTPError as e: print("HTTP error {}".format(e)) response=DummyResponse() response.status=1000 response.data=str(e) return response #a HEAD request def head(self,url): debug=False if debug: print("HEAD: {0}".format(url)) print("as {0}".format(self.auth_name)) headers=urllib3.util.make_headers(basic_auth=self.getBasicAuth()) try: return self.http.request('HEAD',url,headers=headers) except urllib3.exceptions.HTTPError as e: print(e) response=DummyResponse() response.status=1000 response.data=str(e) return response def post(self,url,data): debug=False headers=urllib3.util.make_headers(basic_auth=self.getBasicAuth()) headers["Content-Type"]="application/json" #add csrf;also sets self.cookie headers["X-LABKEY-CSRF"]=self.getCSRF() headers["Cookie"]=self.cookie try: return self.http.request('POST',url,headers=headers,body=data) #f contains json as a return value except urllib3.exceptions.HTTPError as e: print(e) response=DummyResponse() response.status=1000 response.data=str(e) return response def put(self,url,data): debug=False if debug: print("PUT: {}".format(url)) headers=urllib3.util.make_headers(basic_auth=self.getBasicAuth()) headers["Content-Type"]="application/octet-stream" #add csrf headers["X-LABKEY-CSRF"]=self.getCSRF() headers["Cookie"]=self.cookie try: return self.http.request('PUT',url,headers=headers,body=data) #f contains json as a return value except urllib3.exceptions.HTTPError as e: print(e) response=DummyResponse() response.status=1000 response.data=str(e) return response def sendRequest(self,url,requestCode): debug=False headers=urllib3.util.make_headers(basic_auth=self.getBasicAuth()) #add csrf;also sets self.cookie headers["X-LABKEY-CSRF"]=self.getCSRF() headers["Cookie"]=self.cookie try: return self.http.request(requestCode,url,headers=headers) #f contains json as a return value except urllib3.exceptions.HTTPError as e: print(e) response=DummyResponse() response.status=1000 response.data=str(e) return response def mkcol(self,url): return self.sendRequest(url,'MKCOL') def delete(self,url): return self.sendRequest(url,'DELETE') def propfind(self,url,PROPFIND): headers=urllib3.util.make_headers(basic_auth=self.getBasicAuth()) headers["Content-Type"]='text/xml; charset="utf-8"' headers['content-length']=str(len(PROPFIND)) headers['Depth']='1' #add csrf headers["X-LABKEY-CSRF"]=self.getCSRF() headers["Cookie"]=self.cookie try: return self.http.request('PROPFIND',url,headers=headers,body=PROPFIND) #f contains json as a return value except urllib3.exceptions.HTTPError as e: print(e) response=DummyResponse() response.status=1000 response.data=str(e) return response @staticmethod def HTTPStatus(response,method=None): if response.status==200: return True if method=='propfind': if response.status==207: return True print("Status: {}".format(response.status)) print("Data: {}".format(response.data)) return False #a good testing routine; CSRF is required for content modifying requests def getCSRF(self): url=self.GetLabkeyUrl()+'/login/whoAmI.view' try: response=self.get(url) if not labkeyURIHandler.HTTPStatus(response): return None encoding=chardet.detect(response.data)['encoding'] jsonData=json.loads(response.data.decode(encoding)) except AttributeError: print("Response: {}".response.data) print("Failed") return None #local cookie jar self.cookie=response.getheader('Set-Cookie') print("CSRF: {}".format(jsonData["CSRF"])) user=jsonData['email'] if not user==self.connectionConfig['labkey']['user']: print("User mismatch: {}/{}".format(user,self.connectionConfig['labkey']['user'])) return None return jsonData["CSRF"] #file manipulation routiens #was GetFile def DownloadFileToCache(self,relativePath): debug=True # check for file in cache. If available, use, if not, download localPath=self.GetLocalPathFromRelativePath(relativePath) if os.path.isfile(localPath): if debug: print("Returning localFile {}".format(localPath)) return localPath if debug: print("labkeyURIHandler::DownloadFileToCache({0}->{1})".format(relativePath,localPath)) #make all necessary directories LOCALLY path=os.path.dirname(localPath) if debug: print("Target directory: '{}''".format(path)) if not os.path.isdir(path): os.makedirs(path) #make sure we are at the begining of the file #labkeyPath=self.GetLabkeyPathFromRelativePath(relativePath) remoteBuffer=self.readFileToBuffer(relativePath) #check file size if debug: print("Remote size: {}".format(remoteBuffer.seek(0,2))) remoteBuffer.seek(0) with open(localPath,'wb') as localBuffer: shutil.copyfileobj(remoteBuffer,localBuffer) print("Local size: {}".format(os.path.getsize(localPath))) return localPath def fileTypesAvailable(self): return ('VolumeFile','SegmentationFile','TransformFile') #mimic slicer.util.loadNodeFromFile def loadNode(self, relativeName, filetype, properties={},returnNode=False, keepCached=True): #this is the only relevant part - file must be downloaded to cache #labkeyName is just the relative part (from labkey onwards) localPath=self.DownloadFileToCache(relativeName) if not returnNode: slicer.util.loadNodeFromFile(localPath,filetype,properties,returnNode=False) if not keepCached: os.remove(localPath) return ok,node=slicer.util.loadNodeFromFile(localPath,filetype,properties,returnNode=True) if ok: if not keepCached: os.remove(localPath) return node os.remove(localPath) return None # #remove retrieved file def storeNode(self,node,project,dir=None): removeFile=True relativePath=project+'/%40files' if not dir==None: relativePath+='/'+dir labkeyPath=self.GetLabkeyPathFromRelativePath(relativePath) print ("Remote: {}".format(labkeyPath)) #checks if exists self.mkdir(labkeyPath) localPath=self.GetLocalPathFromRelativePath(relativePath) localPath.replace('/',os.path.sep) nodeName=node.GetName() suffix=".nrrd" if node.__class__.__name__=="vtkMRMLDoubleArrayNode": suffix=".mcsv" if node.__class__.__name__=="vtkMRMLTransformNode": suffix=".h5" fileName=nodeName+suffix file=os.path.join(localPath,fileName) slicer.util.saveNode(node,file) print("Stored to: {}".format(file)) f=open(file,"rb") f.seek(0,2) sz=f.tell() print("File size in memory: {0}".format(sz)) f.seek(0) localBuffer=f.read() print("Local buffer size: {}".format(len(localBuffer))) remoteFile=labkeyPath+'/'+fileName resp=self.put(remoteFile,localBuffer) print(resp.read()) f.close() if removeFile: os.remove(file) def entryExists(self,url): #use head response=self.head(url) return labkeyURIHandler.HTTPStatus(response) def remoteDirExists(self,url): #weaker, just checks if there is an entry at url, does not check wheter it is a dir return self.entryExists(url) #stronger, but more complicated return self.isRemoteDir(url) def mkdir(self,remoteDir): if self.remoteDirExists(remoteDir): return False response=self.mkcol(remoteDir) return labkeyURIHandler.HTTPStatus(response) def mkdirs(self,remoteDir): relativePath=self.GetRelativePathFromLabkeyPath(remoteDir) s=0 while True: s1=relativePath.find('/',s) if s1<0: break path=relativePath[0:s1] remotePath=self.GetLabkeyPathFromRelativePath(relativePath) dirExists=self.remoteDirExists(remotePath) if not dirExists: if not self.mkdir(remotePath): return False s=s1+1 return self.mkdir(remoteDir) def rmdir(self,remoteDir): if not self.remoteDirExists(remoteDir): return True return labkeyURIHandler.HTTPStatus(self.delete(remoteDir)) #was listDir def listRelativeDir(self,relativeDir): print("Listing for {0}".format(relativeDir)) dirUrl=self.GetLabkeyPathFromRelativePath(relativeDir) print("Setting url: {}".format(dirUrl)) status,dirs=self.listRemoteDir(dirUrl) dirs=[self.GetRelativePathFromLabkeyPath(d) for d in dirs]; return dirs #was isDir def isRemoteDir(self, remotePath): #print "isDir: {}".format(remotePath) PROPFIND=u"""\n \n \n \n \n """ response=self.propfind(remotePath,PROPFIND) if not labkeyURIHandler.HTTPStatus(response,method='propfind'): print("Bad status") return False try: tree=ET.XML(response.data) except ET.ParseError: #print(f.data.decode('utf-8')) #if directory is not there, a 404 response will be made by HTTP, which is not an xml file #we are safe to assume the directory is not there return False try: rps=tree.find('{DAV:}response').find('{DAV:}propstat') rps=rps.find('{DAV:}prop') rps=rps.find('{DAV:}resourcetype').find('{DAV:}collection') if rps != None: return True else: return False except: return False def listRemoteDir(self,dirUrl): #input is remoteDir, result are remoteDirs PROPFIND=u"""\n \n \n \n \n """ response=self.propfind(dirUrl,PROPFIND) if not labkeyURIHandler.HTTPStatus(response,method='propfind'): print("Bad status") return False,[] try: tree=ET.XML(response.data) except ET.ParseError: print("Fail to parse XML") return False,[] rps=tree.findall('{DAV:}response') dirs=[] for r in rps: hr=r.find('{DAV:}href') dirent=hr.text #dirent=re.sub('/labkey/_webdav/','',dirent) dirent=self.GetHostName()+dirent dirs.append(dirent) del dirs[0] return True,dirs def toRelativePath(self,dirs): flist1=[] for d in dirs: if d[-1]=='/': d=d[:-1] if d.rfind('/')>-1: d=d[d.rfind('/')+1:] flist1.append(d) return flist1 def readFileToBuffer(self, relativePath): dirUrl=self.GetLabkeyPathFromRelativePath(relativePath) response=self.get(dirUrl,binary=True) if not labkeyURIHandler.HTTPStatus(response): return io.BytesIO() #this will collect for all data remoteBuffer=io.BytesIO(response.data) response.release_conn() return remoteBuffer def uploadFile(self,localPath): #get upstream directories sorted out relativePath=self.GetRelativePathFromLocalPath(localPath) if relativePath=="NULL": errorCode="Failed to upload {}. Potential incorrect location" errorCode+=". Should be in labkeyCache!" print(errorCode.format(relativePath)) return False relativeDir=relativePath[0:labkeyPath.rfind('/')] remoteDir=self.GetLabkeyPathFromRemotePath(relativeDir) if not self.remoteDirExists(remoteDir): if not self.mkdirs(remoteDir): errorCode="UploadFile: Could not create directory {}" print(errorCode.format(remoteDir)) return False #make an URL request with open(localPath, 'rb') as f: data=f.read() remotePath=self.GetLabkeyPathFromRelativePath(relativePath) self.put(remotePath,data) #was copyFileToRemote def copyLocalFileToRemote(self,localPath, remotePath): #get upstream directories sorted out remoteDir=os.path.dirname(remotePath) if not self.remoteDirExists(remoteDir): if not self.mkdirs(remoteDir): errorCode="UploadFile: Could not create directory {}" print(errorCode.format(remoteDir)) return False #make an URL request if (self.isRemoteDir(remotePath)): remotePath=remotePath+'/'+os.path.basename(localPath) with open(localPath, 'rb') as f: data=f.read() self.put(remotePath,data) #was loadDir def DownloadDirToCache(self, relativePath): files=self.listRelativeDir(relativePath) fdir="NONE" for f in files: #f is local path try: localDir=os.path.dirname(self.DownloadFileToCache(f)) except: #fails if there is a subdirectory; go recursively print("self.readDir(f) not implemented") return localDir #database routines @staticmethod def getJSON(response): encoding=chardet.detect(response.data)['encoding'] return json.loads(response.data.decode(encoding)) def loadDataset(self,project,dataset): url=self.GetLabkeyUrl()+'/'+project url+='/query-selectRows.api?schemaName=study&query.queryName='+dataset return labkeyURIHandler.getJSON(self.get(url)) def filterDataset(self,project,dataset,filter): debug=False url=self.GetLabkeyUrl()+'/'+project url+='/query-selectRows.api?schemaName=study&query.queryName='+dataset for f in filter: url+="&query."+f['variable']+"~"+f['oper']+"="+f['value'] if debug: print("Sending {}".format(url)) return labkeyURIHandler.getJSON(self.get(url)) def modifyDataset(self,method,project,dataset,rows): #method can be insert or update data={} data['schemaName']='study' data['queryName']=dataset data['rows']=rows url=self.GetLabkeyUrl()+'/'+project url+='/query-'+method+'Rows.api?' return self.post(url,json.dumps(data)).data def filterList(self,project,dataset,filter): schemaName='lists' debug=False url=self.GetLabkeyUrl()+'/'+project url+='/query-selectRows.api?schemaName='+schemaName+'&query.queryName='+dataset for f in filter: url+="&query."+f['variable']+"~"+f['oper']+"="+f['value'] if debug: print("Sending {}".format(url)) return labkeyURIHandler.getJSON(self.get(url)) def modifyList(self,method,project,dataset,rows): #method can be insert or update data={} schemaName='lists' data['schemaName']=schemaName data['queryName']=dataset data['rows']=rows url=self.GetLabkeyUrl()+'/'+project url+='/query-'+method+'Rows.api?' return self.post(url,json.dumps(data)).data