Create subdirectories

Author: amarjandu

hca/dss/__init__.py

@@ -505,7 +505,7 @@ def download_bundle(self, bundle_uuid, version="", metadata_filter=('*',), data_
 
         for file_ in manifest['bundle']['files']:
             dss_file = DSSFile.from_dss_bundle_response(file_, self.replica)
-            filename = file_.get("name", dss_file.uuid)
+            filename = file_.get("name", dss_file.uuid).replace('!', '/')
             walking_dir = bundle_dir
 
             globs = metadata_filter if file_['indexed'] else data_filter

test/integration/dss/test_dss_api.py

@@ -21,12 +21,24 @@
 from test import reset_tweak_changes, TEST_DIR
 
 
+def get_uploaded_file_names(path: str):
+    files = []
+    for entry in os.scandir(path=path):
+        if entry.is_file():
+            files.append(entry.name)
+        if entry.is_dir():
+            nested_files = get_uploaded_file_names(entry.path)
+            for file in nested_files:
+                files.append('{}/{}'.format(entry.name, file))
+    return files
+
+
 class TestDssApi(unittest.TestCase):
     staging_bucket = "org-humancellatlas-dss-cli-test"
 
     @classmethod
     def setUpClass(cls):
-        cls.client = hca.dss.DSSClient()
+        cls.client = hca.dss.DSSClient(swagger_url="https://dss.dev.data.humancellatlas.org/v1/swagger.json")
 
     def test_set_host(self):
         with tempfile.TemporaryDirectory() as home:
@@ -73,7 +85,7 @@ def test_python_nested_bundle_upload_download(self):
 
     def test_python_upload_download(self):
         bundle_path = os.path.join(TEST_DIR, "res", "bundle")
-        uploaded_files = set(os.listdir(bundle_path))
+        uploaded_files = set(get_uploaded_file_names(bundle_path))
 
         manifest = self.client.upload(src_dir=bundle_path,
                                       replica="aws",
@@ -139,7 +151,7 @@ def test_python_upload_download(self):
 
     def test_python_manifest_download(self):
         bundle_path = os.path.join(TEST_DIR, "res", "bundle")
-        uploaded_files = set(os.listdir(bundle_path))
+        uploaded_files = set(get_uploaded_file_names(bundle_path))
 
         manifest = self.client.upload(src_dir=bundle_path,
                                       replica="aws",
@@ -222,9 +234,14 @@ def test_python_bindings(self):
         bundle_path = os.path.join(TEST_DIR, "res", "bundle")
         bundle_output = self.client.upload(src_dir=bundle_path, replica="aws", staging_bucket=self.staging_bucket)
         bundle_uuid = bundle_output['bundle_uuid']
+        bundle_version = bundle_output['version']
+        bundle_fqid = "{}.{}".format(bundle_uuid, bundle_version)
 
         with tempfile.TemporaryDirectory() as dest_dir:
             self.client.download(bundle_uuid=bundle_output['bundle_uuid'], replica="aws", download_dir=dest_dir)
+            nested_downloaded_files = [file.name for file in os.scandir('{}/{}/zarr'.format(dest_dir, bundle_fqid))]
+            for file in ['exclamation.zattrs', 'nested.zattrs']:
+                self.assertIn(file, nested_downloaded_files)
 
         # Test get-files and head-files
         file_ = bundle_output['files'][0]

test/res/bundle/zarr!exclamation.zattrs

@@ -0,0 +1,4 @@
+{
+    "README": "The schema adopted in this zarr store may undergo changes in the future",
+    "sample_id": "0432e9a5-604f-4cb7-8571-014eb5fd8ba2"
+}

test/res/bundle/zarr/nested.zattrs

@@ -0,0 +1,4 @@
+{
+    "README": "The schema adopted in this zarr store may undergo changes in the future",
+    "sample_id": "0432e9a5-604f-4cb7-8571-014eb5fd8ba2"
+}