Scandeps should not recurse on "listing" field because it is copied e… (#198)

* Scandeps handles file and directory literals.  Deduplicate path mapper
inputs so that subdirectories don't accidentally get mapped to a different
location than their parent directories.  Scandeps does not recurse on "listing" or "secondaryFiles" fields because they are copied explicitly.

Author: tetron

cwltool/pathmapper.py

@@ -76,6 +76,27 @@ def abspath(src, basedir):  # type: (Text, Text) -> Text
         ab = src if os.path.isabs(src) else os.path.join(basedir, src)
     return ab
 
+def dedup(listing):  # type: (List[Any]) -> List[Any]
+    marksub = set()
+
+    def mark(d):
+        marksub.add(d["location"])
+
+    for l in listing:
+        if l["class"] == "Directory":
+            for e in l.get("listing", []):
+                adjustFileObjs(e, mark)
+                adjustDirObjs(e, mark)
+
+    dd = []
+    markdup = set()  # type: Set[Text]
+    for r in listing:
+        if r["location"] not in marksub and r["location"] not in markdup:
+            dd.append(r)
+            markdup.add(r["location"])
+
+    return dd
+
 
 class PathMapper(object):
 
@@ -117,7 +138,7 @@ def __init__(self, referenced_files, basedir, stagedir, separateDirs=True):
         self._pathmap = {}  # type: Dict[Text, MapperEnt]
         self.stagedir = stagedir
         self.separateDirs = separateDirs
-        self.setup(referenced_files, basedir)
+        self.setup(dedup(referenced_files), basedir)
 
     def visitlisting(self, listing, stagedir, basedir):
         # type: (List[Dict[Text, Any]], Text, Text) -> None

cwltool/process.py

@@ -647,15 +647,22 @@ def scandeps(base, doc, reffields, urlfields, loadref):
 
         if doc.get("class") in ("File", "Directory") and "location" in urlfields:
             u = doc.get("location", doc.get("path"))
-            if u:
+            if u and not u.startswith("_:"):
                 deps = {
                     "class": doc["class"],
                     "location": urlparse.urljoin(base, u)
                 }
                 if doc["class"] == "Directory" and "listing" in doc:
                     deps["listing"] = doc["listing"]
+                if doc["class"] == "File" and "secondaryFiles" in doc:
+                    deps["secondaryFiles"] = doc["secondaryFiles"]
                 deps = nestdir(base, deps)
                 r.append(deps)
+            else:
+                if doc["class"] == "Directory" and "listing" in doc:
+                    r.extend(scandeps(base, doc["listing"], reffields, urlfields, loadref))
+                elif doc["class"] == "File" and "secondaryFiles" in doc:
+                    r.extend(scandeps(base, doc["secondaryFiles"], reffields, urlfields, loadref))
 
         for k, v in doc.iteritems():
             if k in reffields:
@@ -674,20 +681,21 @@ def scandeps(base, doc, reffields, urlfields, loadref):
                             deps["secondaryFiles"] = sf
                         deps = nestdir(base, deps)
                         r.append(deps)
-            elif k in urlfields:
+            elif k in urlfields and k != "location":
                 for u in aslist(v):
                     deps = {
                         "class": "File",
                         "location": urlparse.urljoin(base, u)
                     }
                     deps = nestdir(base, deps)
                     r.append(deps)
-            else:
+            elif k not in ("listing", "secondaryFiles"):
                 r.extend(scandeps(base, v, reffields, urlfields, loadref))
     elif isinstance(doc, list):
         for d in doc:
             r.extend(scandeps(base, d, reffields, urlfields, loadref))
 
-    normalizeFilesDirs(r)
-    r = mergedirs(r)
+    if r:
+        normalizeFilesDirs(r)
+        r = mergedirs(r)
     return r

tests/test_examples.py

@@ -4,6 +4,7 @@
 import cwltool.expression as expr
 import cwltool.factory
 import cwltool.process
+import cwltool.pathmapper
 import cwltool.workflow
 
 class TestParamMatching(unittest.TestCase):
@@ -134,9 +135,28 @@ def test_scandeps(self):
                                 "location": "file:///example/data2",
                                 "listing": [{
                                     "class": "File",
-                                    "location": "file:///example/data3.txt"
+                                    "location": "file:///example/data3.txt",
+                                    "secondaryFiles": [{
+                                        "class": "File",
+                                        "location": "file:///example/data5.txt"
+                                    }]
+                                }]
+                            },
+                        }, {
+                            "id": "file:///example/bar.cwl#input3",
+                            "default": {
+                                "class": "Directory",
+                                "listing": [{
+                                    "class": "File",
+                                    "location": "file:///example/data4.txt"
                                 }]
                             }
+                        }, {
+                            "id": "file:///example/bar.cwl#input4",
+                            "default": {
+                                "class": "File",
+                                "contents": "file literal"
+                            }
                         }]
                     }
                 }
@@ -173,13 +193,17 @@ def loadref(base, p):
             "listing": [{
                 "basename": "data3.txt",
                 "class": "File",
-                "location": "file:///example/data3.txt"
+                "location": "file:///example/data3.txt",
+                "secondaryFiles": [{
+                    "class": "File",
+                    "basename": "data5.txt",
+                    "location": "file:///example/data5.txt"
+                }]
             }]
-        },
-        {
-            "basename": "data3.txt",
-            "class": "File",
-            "location": "file:///example/data3.txt"
+        }, {
+                "basename": "data4.txt",
+                "class": "File",
+                "location": "file:///example/data4.txt"
         }], sc)
 
         sc = cwltool.process.scandeps(obj["id"], obj,
@@ -194,6 +218,42 @@ def loadref(base, p):
             "location": "file:///example/bar.cwl"
         }], sc)
 
+class TestDedup(unittest.TestCase):
+    def test_dedup(self):
+        ex = [{
+            "class": "File",
+            "location": "file:///example/a"
+        },
+        {
+            "class": "File",
+            "location": "file:///example/a"
+        },
+        {
+            "class": "File",
+            "location": "file:///example/d"
+        },
+        {
+            "class": "Directory",
+            "location": "file:///example/c",
+            "listing": [{
+                "class": "File",
+                "location": "file:///example/d"
+            }]
+        }]
+
+        self.assertEquals([{
+            "class": "File",
+            "location": "file:///example/a"
+        },
+        {
+            "class": "Directory",
+            "location": "file:///example/c",
+            "listing": [{
+                "class": "File",
+                "location": "file:///example/d"
+            }]
+        }], cwltool.pathmapper.dedup(ex))
+
 
 class TestTypeCompare(unittest.TestCase):
     def test_typecompare(self):