Standardization of paths and labels in slicerNetwork

labkeySlicerPythonExtension/fileIO.py

@@ -40,7 +40,7 @@ class remoteFileSelector(qt.QMainWindow):
         self.fileList=qt.QListWidget()
         self.fileList.toolTip="Select remote file"
         self.fileList.itemDoubleClicked.connect(self.onFileListDoubleClicked)
-        self.currentRemoteDir=''
+        self.currentRelativeDir=''
 
         #add dummy entry
         items=('.','..')
@@ -68,31 +68,31 @@ class remoteFileSelector(qt.QMainWindow):
         print ("Selected items: {0} ").format(iText)
 
 
-        #this is hard -> compose path string from currentRemoteDir and selection
+        #this is hard -> compose path string from currentRelativeDir and selection
         if item.text().find('..')==0:
             #one up
-            idx=self.currentRemoteDir.rfind('/')
+            idx=self.currentRelativeDir.rfind('/')
             if idx<0:
-                self.currentRemoteDir=''
+                self.currentRelativeDir=''
             else:
-                self.currentRemoteDir=self.currentRemoteDir[:idx]
+                self.currentRelativeDir=self.currentRelativeDir[:idx]
         elif item.text().find('.')==0:
             pass
         else:
-            if len(self.currentRemoteDir)>0:
-                self.currentRemoteDir+='/'
-            self.currentRemoteDir+=item.text()
+            if len(self.currentRelativeDir)>0:
+                self.currentRelativeDir+='/'
+            self.currentRelativeDir+=item.text()
 
-        print ("Listing {0}").format(self.currentRemoteDir)
+        print ("Listing {0}").format(self.currentRelativeDir)
         flist=self.master.network.toRelativePath(
-            self.master.network.listDir(self.currentRemoteDir))
+            self.master.network.listRelativeDir(self.currentRelativeDir))
         print ("Got")
         print (flist)
         flist.insert(0,'..')
         flist.insert(0,'.')
         self.populateFileList(flist)
-        self.selectedPath.setText(self.currentRemoteDir)
-        self.master.remotePath.setText(self.master.network.GetLabkeyWebdavUrl()+"/"+self.currentRemoteDir)
+        self.selectedPath.setText(self.currentRelativeDir)
+        self.master.remotePath.setText(self.master.network.GetLabkeyPathFromRelativePath(self.currentRelativeDir))
 
     def populateFileList(self,items):
         self.fileList.clear()
@@ -201,24 +201,24 @@ class fileIOWidget(slicer.ScriptedLoadableModule.ScriptedLoadableModuleWidget):
         if os.path.isfile(localPath):
             #end recursion
             print ("Copy {} to {}").format(localPath,remotePath)
-            self.network.copyFileToRemote(localPath,remotePath)
+            self.network.copyLocalFileToRemote(localPath,remotePath)
             return
 
         dirName=os.path.basename(os.path.normpath(localPath))
         remotePath+='/'+dirName
-        if not self.network.isDir(remotePath):
+        if not self.network.isRemoteDir(remotePath):
             self.network.mkdir(remotePath+'/')
         for f in os.listdir(localPath):
             #enter recursion
             localfile=os.path.join(localPath,f)
-            self.copyLocalToRemote(localfile,remotePath)
+            self.copyLocalFileToRemote(localfile,remotePath)
 
     def onCopyToRemoteButtonClicked(self):
-        self.copyLocalToRemote(self.localPath.text,self.remotePath.text)
+        self.copyLocalFileToRemote(self.localPath.text,self.remotePath.text)
 
     def onRemoveRemoteButtonClicked(self):
         remotePath=self.remotePath.text
-        if self.network.isDir(remotePath):
+        if self.network.isRemoteDir(remotePath):
             resp=qt.QMessageBox.question(None,'Delete directory',
                 'Do you want to delete directory {}'.format(remotePath))
             if resp == qt.QMessageBox.No:

labkeySlicerPythonExtension/labkeySlicerPythonExtension.py

@@ -296,7 +296,7 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
             self.currentRemoteDir+=item.text()
         print("Listing {0}").format(self.currentRemoteDir)
         flist=self.network.toRelativePath(
-            self.network.listDir(self.currentRemoteDir))
+            self.network.listRelativeDir(self.currentRemoteDir))
         print("Got")
         print(flist)
         flist.insert(0,'..')
@@ -306,7 +306,7 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
 
   def onLoadFileButtonClicked(self):
       properties={}
-      localPath=self.network.GetFile(self.selectedFile.text)
+      localPath=self.network.DownloadFileToCache(self.selectedFile.text)
       slicer.util.loadNodeFromFile(localPath,self.fileTypeSelector.currentText,
         properties,returnNode=false)
       if not self.keepCachedFileCheckBox.isChecked():

labkeySlicerPythonExtension/loadDicom.py

@@ -38,11 +38,11 @@ class loadDicomLogic(slicer.ScriptedLoadableModule.ScriptedLoadableModuleLogic):
 
     def load(self,sNet,dir,doRemove=True):
         print("Loading dir {}").format(dir)
-        dicomFiles=sNet.listDir(dir)
+        dicomFiles=sNet.listRelativeDir(dir)
         #filelist=[os.path.join(dir,f) for f in os.listdir(dir)]
         filelist=[]
         for f in dicomFiles:
-                localPath=sNet.GetFile(f)
+                localPath=sNet.DownloadFileToCache(f)
                 f0=localPath
                 f1=f0+"1"
                 if not dicomModify==None:

labkeySlicerPythonExtension/slicerNetwork.py

@@ -60,6 +60,7 @@ class labkeyURIHandler(slicer.vtkURIHandler):
     def GetClassName(self):
         return self.className
 
+
     def GetHostName(self):
         return self.hostname
 
@@ -78,120 +79,10 @@ class labkeyURIHandler(slicer.vtkURIHandler):
     def GetLabkeyWebdavUrl(self):
         return self.GetLabkeyUrl()+"/_webdav"
 
-    def GetLocalPath(self,source):
-        debug=False
-        relativePath=re.sub('labkey://','',source)
-        sp=os.sep.encode('string-escape')
-        if debug:
-            print("Substituting / with {0} in {1}").format(sp,relativePath)
-        relativePath=re.sub('/',sp,relativePath)
-        return os.path.join(self.localCacheDirectory,relativePath)
-
-    def GetRemotePath(self,source):
-        return self.GetLabkeyWebdavUrl()+"/"+GetLabkeyPathFromLocalPath(source)
-
-    def GetLabkeyPathFromLocalPath(self,f):
-        #report it with URL separator, forward-slash
-        if f.find(self.localCacheDirectory)<-1:
-            print("Localpath misformation. Exiting")
-            return "NULL"
-        dest=re.sub(self.localCacheDirectory,'',f)
-        #leaves /ContextPath/%40files/subdirectory_list/files
-        #remove first separator
-        sp=os.sep.encode('string-escape')
-        if dest[0]==sp:
-            dest=dest[1:len(dest)]
-        return re.sub(sp,'/',dest)
-
-    def GetLabkeyPathFromRemotePath(self,f):
-        #used to query labkey stuff, so URL separator is used
-        f=re.sub('labkey://','',f)
-        f=re.sub(self.GetLabkeyWebdavUrl(),'',f)
-
-        if f[0]=='/':
-            f=f[1:len(f)]
-        return f;
-
-
-    def GetFile(self,source):
-        # check for file in cache. If available, use, if not, download
-        localPath=self.GetLocalPath(source)
-        if os.path.isfile(localPath):
-            return localPath
-        self.StageFileRead(source,localPath)
-        return localPath
-
-    def StageFileRead(self,source,dest):
-        debug=False
-        if debug:
-            print("labkeyURIHandler::StageFileRead({0},{1})").format(source,dest)
-        labkeyPath=re.sub('labkey://','',source)
-        remote=self.readFile(self.hostname,labkeyPath)
-        #make all necessary directories
-        path=os.path.dirname(dest)
-        if not os.path.isdir(path):
-            os.makedirs(path)
-
-        local=open(dest,'wb')
-        #make sure we are at the begining of the file
-
-	    #check file size
-<<<<<<< HEAD
-	    remote.seek(0,2)
-	    sz=remote.tell()
-	    print "Remote size: {0}".format(sz)
-
-	    remote.seek(0)
-        shutil.copyfileobj(remote,local)
-	    print "Local size: {0}".format(local.tell())
-=======
-        if debug:
-	         remote.seek(0,2)
-	         sz=remote.tell()
-	         print("Remote size: {0}").format(sz)
-
-        remote.seek(0)
-        shutil.copyfileobj(remote,local)
-        if debug:
-	         print("Local size: {0}").format(local.tell())
->>>>>>> 7212758e9e58c9afa6f1d2a0b25eaef3d0a6ad64
-        local.close()
-
-    def StageFileWrite(self,source,dest):
-        print("labkeyURIHandler::StageFileWrite({0},{1}) not implemented yet").format(
-            source,dest)
-
-    def fileTypesAvailable(self):
-        return ('VolumeFile','SegmentationFile','TransformFile')
-
-    #mimic slicer.util.loadNodeFromFile
-    # def loadNodeFromFile(self, filename, filetype, properties={}, returnNode=False):
-    #     #this is the only relevant part - file must be downloaded to cache
-    #     localPath=self.GetFile(filename)
-    #     slicer.util.loadNodeFromFile(localPath,filetype,properties,returnNode)
-    #     #remove retrieved file
-    #     try:
-    #         if not(properties['keepCachedFile']) :
-    #             os.remove(localPath)
-    #     except:
-    #         pass
-
-
-    # def loadVolume(self,filename, properties={}, returnNode=False):
-    #     filetype = 'VolumeFile'
-    #     #redirect to self.loadNodeFromFile first to get the cached file
-    #     return self.loadNodeFromFile(filename,filetype, properties,returnNode)
-    #
-    # def loadSegmentation(self,filename,properties={},returnNode=False):
-    #     filetype='SegmentationFile'
-    #     #redirect to self.loadNodeFromFile first to get the cached file
-    #     return self.loadNodeFromFile(filename,filetype, properties,returnNode)
-    # #add others if needed
-
-    ## setup & initialization routines
+    #configuration part
 
     def configureSSL(self,cert,key,pwd,cacert):
-        #do this first
+    #do this first
         try:
             self.ctx=ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             self.ctx.load_cert_chain(cert,key,pwd)
@@ -241,20 +132,78 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         self.auth_pass=dt['labkey']['password']
 
 
+    #path convention
+
+    #localPath is a path on local filesystem. It means it has to adhere to OS
+    #policy, especially in separators
+
+    #relativePath is just the portion of the path that is equal both locally
+    #and remotely. In relativePath, separator is according to http convention,
+    #which equals separator in osx and *nix
+
+    #labkeyPath or remotePath is the full URL to the resource,
+    #including http-like header and all intermediate portions
+
+    #the following functions convert between different notations
+
+    #was GetLocalPath
+    def GetLocalPathFromRelativePath(self,relativePath):
+        debug=True
+        relativePath=re.sub('labkey://','',relativePath)
+        sp=os.sep.encode('string-escape')
+        if debug:
+            print("Substituting / with {0} in {1}").format(sp,relativePath)
+        relativePath=re.sub('/',sp,relativePath)
+        return os.path.join(self.GetLocalCacheDirectory(),relativePath)
+
+    #was GetRemotePath
+    def GetLabkeyPathFromRelativePath(self,source):
+        return self.GetLabkeyWebdavUrl()+"/"+source
+
+    #was GetLabkeyPathFromLocalPath
+    def GetRelativePathFromLocalPath(self,f):
+        #report it with URL separator, forward-slash
+        if f.find(self.localCacheDirectory)<-1:
+            print("Localpath misformation. Exiting")
+            return "NULL"
+        relativePath=re.sub(self.localCacheDirectory,'',f)
+        #leaves /ContextPath/%40files/subdirectory_list/files
+        #remove first separator
+        sp=os.path.sep
+        if relativePath[0]==sp:
+            relativePath=relativePath[1:len(dest)]
+        return re.sub(sp,'/',relativePath)
+
+    #was GetLabkeyPathFromRemotePath
+    def GetRelativePathFromLabkeyPath(self,f):
+        #used to query labkey stuff, so URL separator is used
+
+        #in old conventions, labkey://was used to tag a labkeyPath
+        f=re.sub('labkey://','',f)
+        f=re.sub(self.GetLabkeyWebdavUrl(),'',f)
+
+        if f[0]=='/':
+            f=f[1:len(f)]
+        return f;
 
-    #cj in opener contains the cookies
 
+    #standard HTTP
     def get(self,url):
 
-        debug=False
+        debug=True
         if debug:
             print("GET: {0}").format(url)
             print("as {0}").format(self.auth_name)
         r=urllib2.Request(url)
         base64string = base64.b64encode('%s:%s' % (self.auth_name, self.auth_pass))
         r.add_header("Authorization", "Basic %s" % base64string)
-        return self.opener.open(r)
+        try:
+            return self.opener.open(r)
         #f contains json as a return value
+        except urllib2.HTTPError as e:
+            print e.code
+            print e.read()
+            return e
 
     def post(self,url,data):
 
@@ -298,6 +247,56 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         return jsonData["CSRF"]
 
 
+    #file manipulation routiens
+
+    #was GetFile
+    def DownloadFileToCache(self,relativePath):
+        debug=False
+        # check for file in cache. If available, use, if not, download
+        localPath=self.GetLocalPathFromRelativePath(relativePath)
+        if os.path.isfile(localPath):
+            return localPath
+
+        if debug:
+            print("labkeyURIHandler::DownloadFileToCache({0}->{1})").format(relativePath,localPath)
+
+        #make all necessary directories LOCALLY
+        path=os.path.dirname(localPath)
+        if not os.path.isdir(path):
+            os.makedirs(path)
+
+        localBuffer=open(localPath,'wb')
+        #make sure we are at the begining of the file
+
+
+        #labkeyPath=self.GetLabkeyPathFromRelativePath(relativePath)
+        remoteBuffer=self.readFileToBuffer(relativePath)
+
+	    #check file size
+        if debug:
+	         remoteBuffer.seek(0,2)
+	         sz=remoteBuffer.tell()
+	         print("Remote size: {0}").format(sz)
+
+        remoteBuffer.seek(0)
+        shutil.copyfileobj(remoteBuffer,localBuffer)
+        if debug:
+	         print("Local size: {0}").format(localBuffer.tell())
+        localBuffer.close()
+
+    def fileTypesAvailable(self):
+        return ('VolumeFile','SegmentationFile','TransformFile')
+
+    #mimic slicer.util.loadNodeFromFile
+    def loadNode(self, relativeName, filetype, properties={}, returnNode=False):
+         #this is the only relevant part - file must be downloaded to cache
+         #labkeyName is just the relative part (from labkey onwards)
+         localPath=self.DownloadFileToCache(relativeName)
+         print localPath
+         slicer.util.loadNodeFromFile(localPath,filetype,properties,returnNode)
+    #     #remove retrieved file
+
+
     def remoteDirExists(self,url):
         status,dirs=self.listRemoteDir(url);
         return status
@@ -316,14 +315,14 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         return True
 
     def mkdirs(self,remoteDir):
-        labkeyPath=self.GetLabkeyPathFromRemotePath(remoteDir)
+        relativePath=self.GetRelativePathFromRemotePath(remoteDir)
         s=0
         while True:
-            s1=labkeyPath.find('/',s)
+            s1=relativePath.find('/',s)
             if s1<0:
                 break
-            path=labkeyPath[0:s1]
-            remotePath=self.GetLabkeyWebdavUrl()+'/'+path
+            path=relativePath[0:s1]
+            remotePath=self.GetLabkeyPathFromRelativePath(relativePath)
             dirExists=self.remoteDirExists(remotePath)
             if not dirExists:
                 if not self.mkdir(remotePath):
@@ -344,13 +343,16 @@ class labkeyURIHandler(slicer.vtkURIHandler):
             return False
         return True
 
-    def listDir(self,dir):
-        print("Listing for {0}").format(dir)
-        dirUrl=self.GetLabkeyWebdavUrl()+"/"+dir
+    #was listDir
+    def listRelativeDir(self,relativeDir):
+        print("Listing for {0}").format(relativeDir)
+        dirUrl=self.GetLabkeyPathFromRelativePath(relativeDir)
         status,dirs=self.listRemoteDir(dirUrl)
+        dirs=[self.GetRelativePathFromLabkeyPath(d) for d in dirs];
         return dirs
 
-    def isDir(self, remotePath):
+    #was isDir
+    def isRemoteDir(self, remotePath):
         #print "isDir: {}".format(remotePath)
         r=MethodRequest(remotePath,method="PROPFIND")
         PROPFIND=u"""<?xml version="1.0" encoding="utf-8"?>\n
@@ -383,6 +385,8 @@ class labkeyURIHandler(slicer.vtkURIHandler):
             return False
 
     def listRemoteDir(self,dirUrl):
+        #input is remoteDir, result are remoteDirs
+
         r=MethodRequest(dirUrl,method="PROPFIND")
         PROPFIND=u"""<?xml version="1.0" encoding="utf-8"?>\n
                     <propfind xmlns="DAV:">\n
@@ -407,7 +411,8 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         for r in rps:
             hr=r.find('{DAV:}href')
             dirent=hr.text
-            dirent=re.sub('/labkey/_webdav/','',dirent)
+            #dirent=re.sub('/labkey/_webdav/','',dirent)
+            dirent=self.GetHostName()+dirent
             dirs.append(dirent)
         del dirs[0]
         return True,dirs
@@ -422,21 +427,21 @@ class labkeyURIHandler(slicer.vtkURIHandler):
             flist1.append(d)
         return flist1
 
-    def readFile(self, serverUrl, path):
-        dirUrl=serverUrl+"/labkey/_webdav"+"/"+path
+    def readFileToBuffer(self, relativePath):
+        dirUrl=self.GetLabkeyPathFromRelativePath(relativePath)
         f=self.get(dirUrl)
         return StringIO.StringIO(f.read())
 
     def uploadFile(self,localPath):
         #get upstream directories sorted out
-        labkeyPath=self.GetLabkeyPathFromLocalPath(localPath)
-        if labkeyPath=="NULL":
+        relativePath=self.GetRelativePathFromLocalPath(localPath)
+        if relativePath=="NULL":
             errorCode="Failed to upload {}. Potential incorrect location"
             errorCode+=". Should be in labkeyCache!"
-            print(errorCode.format(localPath))
+            print(errorCode.format(relativePath))
             return False
-        labkeyDir=labkeyPath[0:labkeyPath.rfind('/')]
-        remoteDir=self.GetLabkeyWebdavUrl()+'/'+labkeyDir
+        relativeDir=relativePath[0:labkeyPath.rfind('/')]
+        remoteDir=self.GetLabkeyPathFromRemotePath(relativeDir)
         if not self.remoteDirExists(remoteDir):
             if not self.mkdirs(remoteDir):
                 errorCode="UploadFile: Could not create directory {}"
@@ -446,39 +451,43 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         #make an URL request
         with open(localPath, 'r') as f:
             data=f.read()
-        remotePath=self.GetLabkeyWebdavUrl()+'/'+labkeyPath
+        remotePath=self.GetLabkeyPathFromRelativePath(relativePath)
         self.put(remotePath,data)
 
-    def copyFileToRemote(self,localPath, remotePath):
+    #was copyFileToRemote
+    def copyLocalFileToRemote(self,localPath, remotePath):
         #get upstream directories sorted out
 
-        labkeyDir=os.path.dirname(remotePath)
-        if not self.remoteDirExists(labkeyDir):
-            if not self.mkdirs(labkeyDir):
+        remoteDir=os.path.dirname(remotePath)
+        if not self.remoteDirExists(remoteDir):
+            if not self.mkdirs(remoteDir):
                 errorCode="UploadFile: Could not create directory {}"
-                print(errorCode.format(labkeyDir))
+                print(errorCode.format(remoteDir))
                 return False
 
         #make an URL request
-        if (self.isDir(remotePath)):
+        if (self.isRemoteDir(remotePath)):
             remotePath=remotePath+'/'+os.path.basename(localPath)
 
         with open(localPath, 'r') as f:
             data=f.read()
         self.put(remotePath,data)
 
-    def loadDir(self, path):
-        #dirURL=serverUrl+"/labkey/_webdav/"+path
-        files=self.listDir(path)
+    #was loadDir
+    def DownloadDirToCache(self, relativePath):
+
+        files=self.listRelativeDir(relativePath)
         fdir="NONE"
         for f in files:
-            #returns local path
+            #f is local path
             try:
-                fdir=os.path.dirname(self.GetFile(f))
+                localDir=os.path.dirname(self.DownloadFileToCache(f))
             except:
                 #fails if there is a subdirectory; go recursively
                 print("self.readDir(f) not implemented")
-        return fdir
+        return localDir
+
+    #database routines
 
     def loadDataset(self,project,dataset):
         url=self.GetLabkeyUrl()+'/'+project