Adding infrastructure to generate multiple entries from a single row

importXLSX.py

@@ -323,6 +323,91 @@ def rowValid(df,r,pars):
 def updateAndValidateEntry(e,pars):
    return True
 
+def generateEntries(df,r,pars):
+   idField=pars.get('idField','ParticipantId')
+   id=getId(df,r,pars)    
+   entry={}
+   entry[idField]=id
+   return [entry]
+
+def setParsingObject(pars)
+   debug=pars.get('debug',0)
+   obj={}
+   obj['fields']=getFields(pars)
+   obj['lookupVars']=getVariables(obj['fields'],fieldType='LOOKUP')
+   dVars=getVariables(obj['fields'],fieldType='DATE')
+   obj['doubleVars']=getVariables(obj['fields'],fieldType='DOUBLE')
+#convert dates to list
+   obj['dateVars']=list(dVars.keys())
+   print(f'dateVars: {dateVars}')
+   obj['lookupMap']={f:getLookupMap(pars,obj['fields'],f) for f in obj['lookupVars']}
+   if debug:
+      print(obj['lookupMap'])
+   alias=getAlias(obj['fields'])
+   obj['alias']=updateAlias(alias,pars)
+   print(f'aliases: {alias}')
+   return obj
+
+
+def setEntry(entry,df,r,objParse,pMap,pars):
+
+   additionalVars=pars.get('additionalVars',{})
+   seqNumOffset=pars.get('seqNumOffset',0)
+   idField=pars.get('idField','ParticipantId')
+   eId=entry[idField]
+
+   for v in df.columns
+      qv=validate(df.at[r,v])
+      qv=modify(qv,v)
+      f=findAlias(v,objParse['alias'])
+      if not f:
+         continue
+      if f in objParse['lookupMap']:
+         if convertLookup(v):
+            qv=parseLookup(objParse['lookupMap'][f],qv)
+         else:
+            qv=asKey(qv,objParse['fields'][f])
+                
+            
+      if f in objParse['dateVars']:
+         qv=parseDate(qv)
+                
+      if f in objParse['doubleVars']:
+         qv=parseDouble(qv)
+      try:
+         numpyType=qv.dtype
+         qv=qv.item()
+      except AttributeError:
+         pass
+            
+      entry[f]=qv
+            
+#print('{}:{}/{}'.format(f,qv,type(qv)))
+
+   for v in additionalVars:
+      f=findAlias(v,objParse['alias'])
+      qv=additionalVars[v]
+      if f in objParse['lookupMap']:
+         qv=parseLookup(objParse['lookupMap'][f],qv)
+      entry[f]=qv
+
+   seqNum=getSequenceNum(pMap,eId)
+   entry['SequenceNum']=seqNum+seqNumOffset
+
+   #set date also? Use pars to find dateVariable
+   try:
+      dateVariable=pars['dateVariable']
+      entry['Date']=entry[dateVariable]
+   except KeyError:
+      pass
+      
+   if not updateAndValidateEntry(entry,pars):
+      return None
+   
+   return entry
+
+
+
 def importData(pars,filename,getId=getId,modify=modify,\
    convertLookup=convertLookup,dryRun=True,debug=True,
    rowValid=rowValid,updateAndValidateEntry=updateAndValidateEntry):
@@ -343,6 +428,7 @@ def importData(pars,filename,getId=getId,modify=modify,\
 # - additionalVars - values that are constant for the whole dataset, labeled by a valid alias for a labkey field
 #- idField - designated unique field of the labkey query
 #- updateAlias - a function to modify aliases based on parameters, e.g. for datasets that take values based on a qualifying field
+# - copyColumns - a dict that adds columns (keys) as new columns (copyColumns[key])
 
    skiprows=pars.get('skiprows',0)
    usecols=pars.get('usecols',None)
@@ -350,24 +436,10 @@ def importData(pars,filename,getId=getId,modify=modify,\
 #set this is as sequenceNum for entries, or initial seqNum if more than a single entry is in the dataset
    seqNumOffset=pars.get('seqNumOffset',0)
    additionalVars=pars.get('additionalVars',{})
-   idField=pars.get('idField','ParticipantId')
    updateAlias=pars.get('updateAlias',lambda x,y: x)
    copyColumns=pars.get('copyColumns',{})
 
-   fields=getFields(pars)
-   lookupVars=getVariables(fields,fieldType='LOOKUP')
-   dateVars=getVariables(fields,fieldType='DATE')
-   doubleVars=getVariables(fields,fieldType='DOUBLE')
-   usecols=pars.get('usecols',None)
-#convert dates to list
-   dateVars=list(dateVars.keys())
-   print(f'dateVars: {dateVars}')
-   lookupMap={f:getLookupMap(pars,fields,f) for f in lookupVars}
-   if debug:
-      print(lookupMap)
-   alias=getAlias(fields)
-   alias=updateAlias(alias,pars)
-   print(f'aliases: {alias}')
+   objParse=setParsingObject(pars)
 
    df=pandas.read_excel(filename,sheet_name=sheet_name,skiprows=skiprows,\
       usecols=usecols)
@@ -375,9 +447,6 @@ def importData(pars,filename,getId=getId,modify=modify,\
    for col in copyColumns:
       df[col]=df[copyColumns[col]]
 
-   vars=df.columns
-   print(vars)
-
    pMap={}
    print('Index: {}'.format(len(df.index)))
    idx=df.index #for all
@@ -388,59 +457,13 @@ def importData(pars,filename,getId=getId,modify=modify,\
    for r in idx:
       if not rowValid(df,r,pars):
          continue
-      id=getId(df,r,pars)    
-      entry={}
-      entry[idField]=id
-
-      for v in vars:
-         qv=validate(df.at[r,v])
-         qv=modify(qv,v)
-         f=findAlias(v,alias)
-         if not f:
+      rowEntries=generateEntries(df,r,pars)
+      for e in rowEntries:
+         e=setEntry(e,df,r,objParse,pMap,pars)
+         if e==None:
             continue
-         if f in lookupMap:
-            if convertLookup(v):
-               qv=parseLookup(lookupMap[f],qv)
-            else:
-               qv=asKey(qv,fields[f])
-                
-            
-         if f in dateVars:
-            qv=parseDate(qv)
-                
-         if f in doubleVars:
-            qv=parseDouble(qv)
-         try:
-            numpyType=qv.dtype
-            qv=qv.item()
-         except AttributeError:
-            pass
-            
-         entry[f]=qv
-            
-#print('{}:{}/{}'.format(f,qv,type(qv)))
-
-      for v in additionalVars:
-         f=findAlias(v,alias)
-         qv=additionalVars[v]
-         if f in lookupMap:
-            qv=parseLookup(lookupMap[f],qv)
-         entry[f]=qv
-
-      seqNum=getSequenceNum(pMap,id)
-      entry['SequenceNum']=seqNum+seqNumOffset
-
-      #set date also? Use pars to find dateVariable
-      try:
-         dateVariable=pars['dateVariable']
-         entry['Date']=entry[dateVariable]
-      except KeyError:
-         pass
-      
-      if not updateAndValidateEntry(entry,pars):
-         continue
 
-      entries.append(entry)
+         entries.append(e)
             
    print(entries)
 #delete previous incarnations