Moving to new path layout

labkeySlicerPythonExtension/fileIO.py

@@ -40,7 +40,7 @@ class remoteFileSelector(qt.QMainWindow):
         self.fileList=qt.QListWidget()
         self.fileList.toolTip="Select remote file"
         self.fileList.itemDoubleClicked.connect(self.onFileListDoubleClicked)
-        self.currentRemoteDir=''
+        self.currentRelativeDir=''
 
         #add dummy entry
         items=('.','..')
@@ -61,38 +61,38 @@ class remoteFileSelector(qt.QMainWindow):
 
     def onFileListDoubleClicked(self,item):
         if item == None:
-            print "Selected items: None"
+            print("Selected items: None")
             return
 
         iText=item.text()
-        print "Selected items: {0} ".format(iText)
+        print ("Selected items: {0} ").format(iText)
 
 
-        #this is hard -> compose path string from currentRemoteDir and selection
+        #this is hard -> compose path string from currentRelativeDir and selection
         if item.text().find('..')==0:
             #one up
-            idx=self.currentRemoteDir.rfind('/')
+            idx=self.currentRelativeDir.rfind('/')
             if idx<0:
-                self.currentRemoteDir=''
+                self.currentRelativeDir=''
             else:
-                self.currentRemoteDir=self.currentRemoteDir[:idx]
+                self.currentRelativeDir=self.currentRelativeDir[:idx]
         elif item.text().find('.')==0:
             pass
         else:
-            if len(self.currentRemoteDir)>0:
-                self.currentRemoteDir+='/'
-            self.currentRemoteDir+=item.text()
+            if len(self.currentRelativeDir)>0:
+                self.currentRelativeDir+='/'
+            self.currentRelativeDir+=item.text()
 
-        print "Listing {0}".format(self.currentRemoteDir)
+        print ("Listing {0}").format(self.currentRelativeDir)
         flist=self.master.network.toRelativePath(
-            self.master.network.listDir(self.currentRemoteDir))
-        print "Got"
-        print flist
+            self.master.network.listRelativeDir(self.currentRelativeDir))
+        print ("Got")
+        print (flist)
         flist.insert(0,'..')
         flist.insert(0,'.')
         self.populateFileList(flist)
-        self.selectedPath.setText(self.currentRemoteDir)
-        self.master.remotePath.setText(self.master.network.GetLabkeyWebdavUrl()+"/"+self.currentRemoteDir)
+        self.selectedPath.setText(self.currentRelativeDir)
+        self.master.remotePath.setText(self.master.network.GetLabkeyPathFromRelativePath(self.currentRelativeDir))
 
     def populateFileList(self,items):
         self.fileList.clear()
@@ -205,18 +205,18 @@ class fileIOWidget(slicer.ScriptedLoadableModule.ScriptedLoadableModuleWidget):
     def copyLocalToRemote(self,localPath,remotePath):
         if os.path.isfile(localPath):
             #end recursion
-            print "Copy {} to {}".format(localPath,remotePath)
-            self.network.copyFileToRemote(localPath,remotePath)
+            print ("Copy {} to {}").format(localPath,remotePath)
+            self.network.copyLocalFileToRemote(localPath,remotePath)
             return
 
         dirName=os.path.basename(os.path.normpath(localPath))
         remotePath+='/'+dirName
-        if not self.network.isDir(remotePath):
+        if not self.network.isRemoteDir(remotePath):
             self.network.mkdir(remotePath+'/')
         for f in os.listdir(localPath):
             #enter recursion
             localfile=os.path.join(localPath,f)
-            self.copyLocalToRemote(localfile,remotePath)
+            self.copyLocalFileToRemote(localfile,remotePath)
 
     def copyRemoteToLocal(self,localPath,remotePath):
         ok,files=self.network.listRemoteDir(remotePath)
@@ -247,14 +247,14 @@ class fileIOWidget(slicer.ScriptedLoadableModule.ScriptedLoadableModuleWidget):
                 fd.close()
 
     def onCopyToRemoteButtonClicked(self):
-        self.copyLocalToRemote(self.localPath.text,self.remotePath.text)
+        self.copyLocalFileToRemote(self.localPath.text,self.remotePath.text)
 
     def onCopyToLocalButtonClicked(self):
         self.copyRemoteToLocal(self.localPath.text,self.remotePath.text)
 
     def onRemoveRemoteButtonClicked(self):
         remotePath=self.remotePath.text
-        if self.network.isDir(remotePath):
+        if self.network.isRemoteDir(remotePath):
             resp=qt.QMessageBox.question(None,'Delete directory',
                 'Do you want to delete directory {}'.format(remotePath))
             if resp == qt.QMessageBox.No:

labkeySlicerPythonExtension/labkeySlicerPythonExtension.py

@@ -50,7 +50,8 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
     self.layout.addWidget(connectionCollapsibleButton)
 
     connectionFormLayout = qt.QFormLayout(connectionCollapsibleButton)
-    self.configDir=os.getenv("HOME")+os.sep+".labkey";
+
+    self.configDir=os.path.join(getHomeDir(),".labkey")
 
     self.serverURL=qt.QLineEdit("https://merlin.fmf.uni-lj.si")
     self.serverURL.textChanged.connect(self.updateServerURL)
@@ -58,12 +59,7 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
     #copy initial setting
     self.updateServerURL(self.serverURL.text);
 
-    try:
-      self.startDir=os.path.join(os.environ['HOME'],"temp/crt")
-    except:
-      fhome=os.environ['HOMEDRIVE']+os.environ['HOMEPATH']
-      self.startDir=os.path.join(fhome,"temp")
-
+    self.startDir=os.path.join(getHomeDir(),"temp/crt")
 
     self.userCertButton=qt.QPushButton("Load")
     self.userCertButton.toolTip="Load user certificate (crt)"
@@ -184,21 +180,21 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
      #pwd=qt.QInputDialog.getText(None,'Certificate password',
      # 'Enter certificate password',qt.QLineEdit.Password)
      if not(filename) :
-         print "No file selected"
+         print("No file selected")
          return
 
      f=qt.QFile(filename)
      if not (f.open(qt.QIODevice.ReadOnly)) :
-         print "Could not open file"
+         print("Could not open file")
          return
 
      certList=qt.QSslCertificate.fromPath(filename)
      if len(certList) < 1:
-         print "Troubles parsing {0}".format(filename)
+         print ("Troubles parsing {0}").format(filename)
          return
 
      self.logic.cert=qt.QSslCertificate(f)
-     print "cert.isNull()={0}".format(self.logic.cert.isNull())
+     print("cert.isNull()={0}").format(self.logic.cert.isNull())
      self.userCertButton.setText(filename)
      self.authName.setText(self.logic.cert.subjectInfo("emailAddress"))
 
@@ -206,12 +202,12 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
       filename=qt.QFileDialog.getOpenFileName(None,'Open private key',
             self.startDir, '*.key')
       if not (filename) :
-          print "No file selected"
+          print ("No file selected")
           return
 
       f=qt.QFile(filename)
       if not (f.open(qt.QIODevice.ReadOnly)) :
-          print "Could not open file"
+          print ("Could not open file")
           return
       self.pwd=qt.QInputDialog.getText(None,'Private key password',
         'Enter key password',qt.QLineEdit.Password)
@@ -224,19 +220,19 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
       filename=qt.QFileDialog.getOpenFileName(None,'Open authority certificate',
                self.startDir, '*.crt')
       if not(filename) :
-         print "No file selected"
+         print("No file selected")
          return
 
       f=qt.QFile(filename)
 
       if not (f.open(qt.QIODevice.ReadOnly)) :
-          print "Could not open file"
+          print("Could not open file")
           return
 
       certList=qt.QSslCertificate.fromPath(filename)
 
       if len(certList) < 1:
-          print "Troubles parsing {0}".format(filename)
+          print("Troubles parsing {0}").format(filename)
           return
       self.logic.caCert=qt.QSslCertificate(f)#certList[0]
       self.caCertButton.setText(filename)
@@ -244,63 +240,44 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
   def onLoadConfigButtonClicked(self):
        filename=qt.QFileDialog.getOpenFileName(None,'Open configuration file (JSON)',
             self.configDir, '*.json')
-       with open(filename,'r') as f:
-           dt=json.load(f)
-       if dt.has_key('host'):
-           self.serverURL.setText(dt['host'])
-       if dt.has_key('dataset'):
-           pass
-           #self.datasetName.setText(dt['dataset'])
-       if dt.has_key('project'):
-           pass
-           #self.datasetProject.setText(dt['project'])
-       if dt.has_key('SSL'):
-           if dt['SSL'].has_key('user'):
-               self.userCertButton.setText(dt['SSL']['user'])
-           if dt['SSL'].has_key('key'):
-               self.privateKeyButton.setText(dt['SSL']['key'])
-           if dt['SSL'].has_key('keyPwd'):
-               self.pwd=dt['SSL']['keyPwd']
-           if dt['SSL'].has_key('ca'):
-               self.caCertButton.setText(dt['SSL']['ca'])
-       if dt.has_key('labkey'):
-           if dt['labkey'].has_key('user'):
-               self.network.auth_name=dt['labkey']['user']
-               self.authName.setText(self.network.auth_name)
-           if dt['labkey'].has_key('password'):
-               self.network.auth_pass=dt['labkey']['password']
-               self.authPass.setText(self.network.auth_pass)
+       self.network.parseConfig(filename)
+       self.serverURL.setText(self.network.hostname)
+       self.authName.setText(self.network.auth_name)
+       self.authPass.setText(self.network.auth_pass)
+
        self.loadConfigButton.setText(os.path.basename(filename))
 
   def onInitButtonClicked(self):
-      self.network.configureSSL(
-        self.userCertButton.text,
-        self.privateKeyButton.text,
-        self.pwd,
-        self.caCertButton.text
-      )
+      if self.serverURL.text.find("https")==0:
+
+          self.network.configureSSL(
+            self.userCertButton.text,
+            self.privateKeyButton.text,
+            self.pwd,
+            self.caCertButton.text
+        )
 
       self.network.initRemote()
 
   def updateAuthName(self,txt):
       self.network.auth_name=txt
-      print "Setting username to {0}".format(self.network.auth_name);
+      print("Setting username to {0}").format(self.network.auth_name);
 
   def updateAuthPass(self,txt):
       self.network.auth_pass=txt
-      print "Setting password."
+      print("Setting password.")
 
   def updateServerURL(self,txt):
       self.network.hostname=txt
-      print "Setting hostname to {0}".format(self.network.hostname);
+      print("Setting hostname to {0}").format(self.network.hostname);
 
   def onFileListDoubleClicked(self,item):
         if item == None:
-            print "Selected items: None"
+            print("Selected items: None")
             return
 
         iText=item.text()
-        print "Selected items: {0} ".format(iText)
+        print("Selected items: {0} ").format(iText)
 
 
         #this is hard -> compose path string from currentRemoteDir and selection
@@ -317,11 +294,11 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
             if len(self.currentRemoteDir)>0:
                 self.currentRemoteDir+='/'
             self.currentRemoteDir+=item.text()
-        print "Listing {0}".format(self.currentRemoteDir)
+        print("Listing {0}").format(self.currentRemoteDir)
         flist=self.network.toRelativePath(
-            self.network.listDir(self.currentRemoteDir))
-        print "Got"
-        print flist
+            self.network.listRelativeDir(self.currentRemoteDir))
+        print("Got")
+        print(flist)
         flist.insert(0,'..')
         flist.insert(0,'.')
         self.populateFileList(flist)
@@ -329,19 +306,20 @@ class labkeySlicerPythonExtensionWidget(ScriptedLoadableModuleWidget):
 
   def onLoadFileButtonClicked(self):
       properties={}
-      localPath=self.network.GetFile(self.selectedFile.text)
+      localPath=self.network.DownloadFileToCache(self.selectedFile.text)
       slicer.util.loadNodeFromFile(localPath,self.fileTypeSelector.currentText,
         properties,returnNode=false)
       if not self.keepCachedFileCheckBox.isChecked():
               os.remove(localPath)
 
   def onLoadDirButtonClicked(self):
-    localDir=self.network.loadDir(self.selectedFile.text)
+    #dir=self.network.loadDir(self.selectedFile.text)
+    dir=self.selectedFile.text
     try:
-        self.loadDicomLogic.load(localDir)
+        self.loadDicomLogic.load(self.network,dir)
     except:
-        self.loadDicom=loadDicom.loadDicomLogic(self)
-        self.loadDicom.load(localDir)
+        self.loadDicomLogic=loadDicom.loadDicomLogic(self)
+        self.loadDicomLogic.load(self.network,dir)
 
 # labkeySlicerPythonExtensionLogic
 #
@@ -480,3 +458,10 @@ class labkeySlicerPythonExtensionTest(ScriptedLoadableModuleTest):
     logic = labkeySlicerPythonExtensionLogic()
     self.assertTrue( logic.hasImageData(volumeNode) )
     self.delayDisplay('Test passed!')
+
+def getHomeDir():
+    try:
+      return os.environ['HOME']
+    except:
+      fhome=os.environ['HOMEDRIVE']+os.environ['HOMEPATH']
+      return fhome

labkeySlicerPythonExtension/loadDicom.py

@@ -4,8 +4,9 @@ import subprocess
 import re
 
 dicomModify=os.getenv("HOME")
-dicomModify+="/software/install/"
-dicomModify+="dicomModify/bin/dicomModify"
+if not dicomModify==None:
+    dicomModify+="/software/install/"
+    dicomModify+="dicomModify/bin/dicomModify"
 
 class loadDicom(slicer.ScriptedLoadableModule.ScriptedLoadableModule):
     def __init__(self,parent):
@@ -36,14 +37,16 @@ class loadDicomLogic(slicer.ScriptedLoadableModule.ScriptedLoadableModuleLogic):
       slicer.ScriptedLoadableModule.ScriptedLoadableModuleLogic.__init__(self, parent)
 
     def load(self,sNet,dir,doRemove=True):
-        dicomFiles=sNet.listDir(dir)
+        print("Loading dir {}").format(dir)
+        dicomFiles=sNet.listRelativeDir(dir)
         #filelist=[os.path.join(dir,f) for f in os.listdir(dir)]
         filelist=[]
         for f in dicomFiles:
-                localPath=sNet.GetFile(f)
+                localPath=sNet.DownloadFileToCache(f)
                 f0=localPath
                 f1=f0+"1"
-                subprocess.call(dicomModify+" "+f0+" "+f1+" && mv "+f1+" "+f0+";", shell=True)
+                if not dicomModify==None:
+                    subprocess.call(dicomModify+" "+f0+" "+f1+" && mv "+f1+" "+f0+";", shell=True)
                 filelist.append(localPath)
 
         try:

labkeySlicerPythonExtension/slicerNetwork.py

@@ -39,10 +39,10 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         self.className="labkeyURIHandler"
         slicer.mrmlScene.AddURIHandler(self)
         try:
-	       fhome=os.environ["HOME"]
+            fhome=os.environ["HOME"]
         except:
-	    #in windows, the variable is called HOMEPATH
-	       fhome=os.environ['HOMEDRIVE']+os.environ['HOMEPATH']
+	           #in windows, the variable is called HOMEPATH
+            fhome=os.environ['HOMEDRIVE']+os.environ['HOMEPATH']
 
         self.localCacheDirectory=os.path.join(fhome,"labkeyCache")
         self.configDir=os.path.join(fhome,".labkey")
@@ -52,7 +52,7 @@ class labkeyURIHandler(slicer.vtkURIHandler):
 
 
     def CanHandleURI(self,uri):
-        print "labkeyURIHandler::CanHandleURI({0})".format(uri)
+        print("labkeyURIHandler::CanHandleURI({0})").format(uri)
         if uri.find('labkey://')==0:
             return 1
         return 0
@@ -60,6 +60,7 @@ class labkeyURIHandler(slicer.vtkURIHandler):
     def GetClassName(self):
         return self.className
 
+
     def GetHostName(self):
         return self.hostname
 
@@ -78,117 +79,17 @@ class labkeyURIHandler(slicer.vtkURIHandler):
     def GetLabkeyWebdavUrl(self):
         return self.GetLabkeyUrl()+"/_webdav"
 
-    def GetLocalPath(self,source):
-        debug=False
-        relativePath=re.sub('labkey://','',source)
-        sp=os.sep.encode('string-escape')
-        if debug:
-            print "Substituting / with {0} in {1}".format(sp,relativePath)
-        relativePath=re.sub('/',sp,relativePath)
-        return os.path.join(self.localCacheDirectory,relativePath)
-
-    def GetRemotePath(self,source):
-        return self.GetLabkeyWebdavUrl()+"/"+GetLabkeyPathFromLocalPath(source)
-
-    def GetLabkeyPathFromLocalPath(self,f):
-        #report it with URL separator, forward-slash
-        if f.find(self.localCacheDirectory)<-1:
-            print "Localpath misformation. Exiting"
-            return "NULL"
-        dest=re.sub(self.localCacheDirectory,'',f)
-        #leaves /ContextPath/%40files/subdirectory_list/files
-        #remove first separator
-        sp=os.sep.encode('string-escape')
-        if dest[0]==sp:
-            dest=dest[1:len(dest)]
-        return re.sub(sp,'/',dest)
-
-    def GetLabkeyPathFromRemotePath(self,f):
-        #used to query labkey stuff, so URL separator is used
-        f=re.sub('labkey://','',f)
-        f=re.sub(self.GetLabkeyWebdavUrl(),'',f)
-
-        if f[0]=='/':
-            f=f[1:len(f)]
-        return f;
-
-
-    def GetFile(self,source):
-        # check for file in cache. If available, use, if not, download
-        localPath=self.GetLocalPath(source)
-        if os.path.isfile(localPath):
-            return localPath
-        self.StageFileRead(source,localPath)
-        return localPath
-
-    def StageFileRead(self,source,dest):
-        debug=False
-        if debug:
-            print "labkeyURIHandler::StageFileRead({0},{1})".format(source,dest)
-        labkeyPath=re.sub('labkey://','',source)
-        remote=self.readFile(labkeyPath)
-        #make all necessary directories
-        path=os.path.dirname(dest)
-        if not os.path.isdir(path):
-            os.makedirs(path)
-
-        local=open(dest,'wb')
-        #make sure we are at the begining of the file
-
-	    #check file size
-        if debug:
-	         remote.seek(0,2)
-	         sz=remote.tell()
-	         print "Remote size: {0}".format(sz)
-
-        remote.seek(0)
-        shutil.copyfileobj(remote,local)
-        if debug:
-	         print "Local size: {0}".format(local.tell())
-        local.close()
-
-    def StageFileWrite(self,source,dest):
-        print "labkeyURIHandler::StageFileWrite({0},{1}) not implemented yet".format(
-            source,dest)
-
-    def fileTypesAvailable(self):
-        return ('VolumeFile','SegmentationFile','TransformFile')
-
-    #mimic slicer.util.loadNodeFromFile
-    # def loadNodeFromFile(self, filename, filetype, properties={}, returnNode=False):
-    #     #this is the only relevant part - file must be downloaded to cache
-    #     localPath=self.GetFile(filename)
-    #     slicer.util.loadNodeFromFile(localPath,filetype,properties,returnNode)
-    #     #remove retrieved file
-    #     try:
-    #         if not(properties['keepCachedFile']) :
-    #             os.remove(localPath)
-    #     except:
-    #         pass
-
-
-    # def loadVolume(self,filename, properties={}, returnNode=False):
-    #     filetype = 'VolumeFile'
-    #     #redirect to self.loadNodeFromFile first to get the cached file
-    #     return self.loadNodeFromFile(filename,filetype, properties,returnNode)
-    #
-    # def loadSegmentation(self,filename,properties={},returnNode=False):
-    #     filetype='SegmentationFile'
-    #     #redirect to self.loadNodeFromFile first to get the cached file
-    #     return self.loadNodeFromFile(filename,filetype, properties,returnNode)
-    # #add others if needed
-
-    ## setup & initialization routines
+    #configuration part
 
     def configureSSL(self,cert,key,pwd,cacert):
-        #do this first
+    #do this first
         try:
             self.ctx=ssl.SSLContext(ssl.PROTOCOL_SSLv23)
             self.ctx.load_cert_chain(cert,key,pwd)
             self.ctx.verify_mode=ssl.CERT_REQUIRED
             self.ctx.load_verify_locations(cacert)
         except ssl.SSLError as err:
-            print " Failed to configure SSL: {0}".format(str(err))
+            print(" Failed to configure SSL: {0}").format(str(err))
         self.mode="https"
 
 
@@ -203,7 +104,7 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         self.opener=urllib2.build_opener(http_handler,cookie_handler)
 
     def initFromConfig(self):
-        path=os.path.join(self.configDir,"remote.json")
+        path=os.path.join(self.configDir,"Remote.json")
         try:
             self.parseConfig(path)
         except OSError:
@@ -214,10 +115,11 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         try:
             f=open(fname)
         except OSError as e:
-            print "Confgiuration error: OS error({0}): {1}".format(e.errno, e.strerror)
+            print("Confgiuration error: OS error({0}): {1}").format(e.errno, e.strerror)
             raise
 
         dt=json.load(f)
+        self.mode="http"
         if dt.has_key('SSL'):
             self.configureSSL(
                 dt['SSL']['user'],
@@ -230,20 +132,78 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         self.auth_pass=dt['labkey']['password']
 
 
+    #path convention
+
+    #localPath is a path on local filesystem. It means it has to adhere to OS
+    #policy, especially in separators
+
+    #relativePath is just the portion of the path that is equal both locally
+    #and remotely. In relativePath, separator is according to http convention,
+    #which equals separator in osx and *nix
+
+    #labkeyPath or remotePath is the full URL to the resource,
+    #including http-like header and all intermediate portions
+
+    #the following functions convert between different notations
+
+    #was GetLocalPath
+    def GetLocalPathFromRelativePath(self,relativePath):
+        debug=True
+        relativePath=re.sub('labkey://','',relativePath)
+        sp=os.sep.encode('string-escape')
+        if debug:
+            print("Substituting / with {0} in {1}").format(sp,relativePath)
+        relativePath=re.sub('/',sp,relativePath)
+        return os.path.join(self.GetLocalCacheDirectory(),relativePath)
 
-    #cj in opener contains the cookies
+    #was GetRemotePath
+    def GetLabkeyPathFromRelativePath(self,source):
+        return self.GetLabkeyWebdavUrl()+"/"+source
 
+    #was GetLabkeyPathFromLocalPath
+    def GetRelativePathFromLocalPath(self,f):
+        #report it with URL separator, forward-slash
+        if f.find(self.localCacheDirectory)<-1:
+            print("Localpath misformation. Exiting")
+            return "NULL"
+        relativePath=re.sub(self.localCacheDirectory,'',f)
+        #leaves /ContextPath/%40files/subdirectory_list/files
+        #remove first separator
+        sp=os.path.sep
+        if relativePath[0]==sp:
+            relativePath=relativePath[1:len(dest)]
+        return re.sub(sp,'/',relativePath)
+
+    #was GetLabkeyPathFromRemotePath
+    def GetRelativePathFromLabkeyPath(self,f):
+        #used to query labkey stuff, so URL separator is used
+
+        #in old conventions, labkey://was used to tag a labkeyPath
+        f=re.sub('labkey://','',f)
+        f=re.sub(self.GetLabkeyWebdavUrl(),'',f)
+
+        if f[0]=='/':
+            f=f[1:len(f)]
+        return f;
+
+
+    #standard HTTP
     def get(self,url):
 
-        debug=False
+        debug=True
         if debug:
-            print "GET: {0}".format(url)
-            print "as {0}".format(self.auth_name)
+            print("GET: {0}").format(url)
+            print("as {0}").format(self.auth_name)
         r=urllib2.Request(url)
         base64string = base64.b64encode('%s:%s' % (self.auth_name, self.auth_pass))
         r.add_header("Authorization", "Basic %s" % base64string)
-        return self.opener.open(r)
+        try:
+            return self.opener.open(r)
         #f contains json as a return value
+        except urllib2.HTTPError as e:
+            print e.code
+            print e.read()
+            return e
 
     def post(self,url,data):
 
@@ -257,15 +217,20 @@ class labkeyURIHandler(slicer.vtkURIHandler):
 
         base64string = base64.b64encode('%s:%s' % (self.auth_name, self.auth_pass))
         r.add_header("Authorization", "Basic %s" % base64string)
-        print "{}: {}".format(r.get_method(),r.get_full_url())
-        print "data: {}".format(r.get_data())
-        print "Content-Type: {}".format(r.get_header('Content-Type'))
-        return self.opener.open(r)
+        print("{}: {}").format(r.get_method(),r.get_full_url())
+        print("data: {}").format(r.get_data())
+        print("Content-Type: {}").format(r.get_header('Content-Type'))
+        try:
+            return self.opener.open(r)
+        except urllib2.HTTPError as e:
+            print e.code
+            print e.read()
+            return e
         #f contains json as a return value
 
     def put(self,url,data):
 
-        print "PUT: {0}".format(url)
+        print("PUT: {0}").format(url)
         r=MethodRequest(url.encode('utf-8'),method="PUT")
         #makes it a post
         r.add_data(data)
@@ -282,6 +247,56 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         return jsonData["CSRF"]
 
 
+    #file manipulation routiens
+
+    #was GetFile
+    def DownloadFileToCache(self,relativePath):
+        debug=False
+        # check for file in cache. If available, use, if not, download
+        localPath=self.GetLocalPathFromRelativePath(relativePath)
+        if os.path.isfile(localPath):
+            return localPath
+
+        if debug:
+            print("labkeyURIHandler::DownloadFileToCache({0}->{1})").format(relativePath,localPath)
+
+        #make all necessary directories LOCALLY
+        path=os.path.dirname(localPath)
+        if not os.path.isdir(path):
+            os.makedirs(path)
+
+        localBuffer=open(localPath,'wb')
+        #make sure we are at the begining of the file
+
+
+        #labkeyPath=self.GetLabkeyPathFromRelativePath(relativePath)
+        remoteBuffer=self.readFileToBuffer(relativePath)
+
+	    #check file size
+        if debug:
+	         remoteBuffer.seek(0,2)
+	         sz=remoteBuffer.tell()
+	         print("Remote size: {0}").format(sz)
+
+        remoteBuffer.seek(0)
+        shutil.copyfileobj(remoteBuffer,localBuffer)
+        if debug:
+	         print("Local size: {0}").format(localBuffer.tell())
+        localBuffer.close()
+
+    def fileTypesAvailable(self):
+        return ('VolumeFile','SegmentationFile','TransformFile')
+
+    #mimic slicer.util.loadNodeFromFile
+    def loadNode(self, relativeName, filetype, properties={}, returnNode=False):
+         #this is the only relevant part - file must be downloaded to cache
+         #labkeyName is just the relative part (from labkey onwards)
+         localPath=self.DownloadFileToCache(relativeName)
+         print localPath
+         slicer.util.loadNodeFromFile(localPath,filetype,properties,returnNode)
+    #     #remove retrieved file
+
+
     def remoteDirExists(self,url):
         status,dirs=self.listRemoteDir(url);
         return status
@@ -295,19 +310,19 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         try:
             f=self.opener.open(r)
         except:
-            print "Error: Failed MKCOL {}".format(remoteDir)
+            print("Error: Failed MKCOL {}").format(remoteDir)
             return False
         return True
 
     def mkdirs(self,remoteDir):
-        labkeyPath=self.GetLabkeyPathFromRemotePath(remoteDir)
+        relativePath=self.GetRelativePathFromRemotePath(remoteDir)
         s=0
         while True:
-            s1=labkeyPath.find('/',s)
+            s1=relativePath.find('/',s)
             if s1<0:
                 break
-            path=labkeyPath[0:s1]
-            remotePath=self.GetLabkeyWebdavUrl()+'/'+path
+            path=relativePath[0:s1]
+            remotePath=self.GetLabkeyPathFromRelativePath(relativePath)
             dirExists=self.remoteDirExists(remotePath)
             if not dirExists:
                 if not self.mkdir(remotePath):
@@ -324,17 +339,20 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         try:
             f=self.opener.open(r)
         except:
-            print "Error: Failed DELETE {}".format(remoteDir)
+            print("Error: Failed DELETE {}").format(remoteDir)
             return False
         return True
 
-    def listDir(self,dir):
-        print "Listing for {0}".format(dir)
-        dirUrl=self.GetLabkeyWebdavUrl()+"/"+dir
+    #was listDir
+    def listRelativeDir(self,relativeDir):
+        print("Listing for {0}").format(relativeDir)
+        dirUrl=self.GetLabkeyPathFromRelativePath(relativeDir)
         status,dirs=self.listRemoteDir(dirUrl)
+        dirs=[self.GetRelativePathFromLabkeyPath(d) for d in dirs];
         return dirs
 
-    def isDir(self, remotePath):
+    #was isDir
+    def isRemoteDir(self, remotePath):
         #print "isDir: {}".format(remotePath)
         r=MethodRequest(remotePath,method="PROPFIND")
         PROPFIND=u"""<?xml version="1.0" encoding="utf-8"?>\n
@@ -349,7 +367,7 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         r.add_data(PROPFIND)
         base64string = base64.b64encode('%s:%s' % (self.auth_name, self.auth_pass))
         r.add_header("Authorization", "Basic %s" % base64string)
-        print "PROPFIND: {0}".format(remotePath)
+        print("PROPFIND: {0}").format(remotePath)
         try:
             f=self.opener.open(r)
         except:
@@ -367,6 +385,8 @@ class labkeyURIHandler(slicer.vtkURIHandler):
             return False
 
     def listRemoteDir(self,dirUrl):
+        #input is remoteDir, result are remoteDirs
+
         r=MethodRequest(dirUrl,method="PROPFIND")
         PROPFIND=u"""<?xml version="1.0" encoding="utf-8"?>\n
                     <propfind xmlns="DAV:">\n
@@ -380,7 +400,7 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         r.add_data(PROPFIND)
         base64string = base64.b64encode('%s:%s' % (self.auth_name, self.auth_pass))
         r.add_header("Authorization", "Basic %s" % base64string)
-        print "PROPFIND: {0}".format(dirUrl)
+        print("PROPFIND: {0}").format(dirUrl)
         dirs=[]
         try:
             f=self.opener.open(r)
@@ -391,7 +411,8 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         for r in rps:
             hr=r.find('{DAV:}href')
             dirent=hr.text
-            dirent=re.sub('/labkey/_webdav/','',dirent)
+            #dirent=re.sub('/labkey/_webdav/','',dirent)
+            dirent=self.GetHostName()+dirent
             dirs.append(dirent)
         del dirs[0]
         return True,dirs
@@ -406,64 +427,67 @@ class labkeyURIHandler(slicer.vtkURIHandler):
             flist1.append(d)
         return flist1
 
-    def readFile(self, path):
-        dirUrl=self.GetLabkeyWebdavUrl()+"/"+path
-        print 'Reading {}'.format(dirUrl)
+    def readFileToBuffer(self, relativePath):
+        dirUrl=self.GetLabkeyPathFromRelativePath(relativePath)
         f=self.get(dirUrl)
         return StringIO.StringIO(f.read())
 
     def uploadFile(self,localPath):
         #get upstream directories sorted out
-        labkeyPath=self.GetLabkeyPathFromLocalPath(localPath)
-        if labkeyPath=="NULL":
+        relativePath=self.GetRelativePathFromLocalPath(localPath)
+        if relativePath=="NULL":
             errorCode="Failed to upload {}. Potential incorrect location"
             errorCode+=". Should be in labkeyCache!"
-            print errorCode.format(localPath)
+            print(errorCode.format(relativePath))
             return False
-        labkeyDir=labkeyPath[0:labkeyPath.rfind('/')]
-        remoteDir=self.GetLabkeyWebdavUrl()+'/'+labkeyDir
+        relativeDir=relativePath[0:labkeyPath.rfind('/')]
+        remoteDir=self.GetLabkeyPathFromRemotePath(relativeDir)
         if not self.remoteDirExists(remoteDir):
             if not self.mkdirs(remoteDir):
                 errorCode="UploadFile: Could not create directory {}"
-                print errorCode.format(remoteDir)
+                print(errorCode.format(remoteDir))
                 return False
 
         #make an URL request
         with open(localPath, 'r') as f:
             data=f.read()
-        remotePath=self.GetLabkeyWebdavUrl()+'/'+labkeyPath
+        remotePath=self.GetLabkeyPathFromRelativePath(relativePath)
         self.put(remotePath,data)
 
-    def copyFileToRemote(self,localPath, remotePath):
+    #was copyFileToRemote
+    def copyLocalFileToRemote(self,localPath, remotePath):
         #get upstream directories sorted out
 
-        labkeyDir=os.path.dirname(remotePath)
-        if not self.remoteDirExists(labkeyDir):
-            if not self.mkdirs(labkeyDir):
+        remoteDir=os.path.dirname(remotePath)
+        if not self.remoteDirExists(remoteDir):
+            if not self.mkdirs(remoteDir):
                 errorCode="UploadFile: Could not create directory {}"
-                print errorCode.format(labkeyDir)
+                print(errorCode.format(remoteDir))
                 return False
 
         #make an URL request
-        if (self.isDir(remotePath)):
+        if (self.isRemoteDir(remotePath)):
             remotePath=remotePath+'/'+os.path.basename(localPath)
 
         with open(localPath, 'r') as f:
             data=f.read()
         self.put(remotePath,data)
 
-    def loadDir(self, path):
-        #dirURL=serverUrl+"/labkey/_webdav/"+path
-        files=self.listDir(path)
+    #was loadDir
+    def DownloadDirToCache(self, relativePath):
+
+        files=self.listRelativeDir(relativePath)
         fdir="NONE"
         for f in files:
-            #returns local path
+            #f is local path
             try:
-                fdir=os.path.dirname(self.GetFile(f))
+                localDir=os.path.dirname(self.DownloadFileToCache(f))
             except:
                 #fails if there is a subdirectory; go recursively
-                print "self.readDir(f) not implemented"
-        return fdir
+                print("self.readDir(f) not implemented")
+        return localDir
+
+    #database routines
 
     def loadDataset(self,project,dataset):
         url=self.GetLabkeyUrl()+'/'+project
@@ -476,7 +500,7 @@ class labkeyURIHandler(slicer.vtkURIHandler):
         url+='/query-selectRows.api?schemaName=study&query.queryName='+dataset
         url+="&query."+variable+"~"+oper+"="+value
         if debug:
-            print "Sending {}".format(url)
+            print("Sending {}").format(url)
         return json.load(self.get(url))