Merge pull request #2131 from AllenInstitute/bugfix/create_ecephys_up…

…load_dir

tests for production mkdir error and fixes

CHANGELOG.md

@@ -2,6 +2,9 @@
 All notable changes to this project will be documented in this file.
 
 
+## [2.11.2] = 2021-05-21
+- Fixed mkdir error for non-existing ecephys upload directory
+
 ## [2.11.1] = 2021-05-20
 - Refactored the schema for the ecephys copy utility to avoid raising an error when a previous output file already exists.
 

allensdk/__init__.py

@@ -35,7 +35,7 @@
 #
 import logging
 
-__version__ = '2.11.1'
+__version__ = '2.11.2'
 
 
 try:

allensdk/brain_observatory/ecephys/copy_utility/_schemas.py

@@ -1,8 +1,7 @@
 import hashlib
-
 from argschema import ArgSchema
 from argschema.fields import (
-    LogLevel, String, Int, Nested, Boolean, List, InputFile, OutputFile)
+    LogLevel, String, Int, Nested, Boolean, List, InputFile)
 from argschema.schemas import DefaultSchema
 
 
@@ -18,10 +17,12 @@ class FileExists(InputFile):
 
 
 class FileToCopy(DefaultSchema):
-    source = InputFile(required=True, description='copy from here')
-    destination = OutputFile(required=True,
-                             description='copy to here (full path, not just '
-                                         'directory!)')
+    source = InputFile(
+            required=True,
+            description='copy from here')
+    destination = String(
+            required=True,
+            description='copy to here (full path, not just directory!)')
     key = String(required=True,
                  description='will be passed through to outputs, allowing a '
                              'name or kind to be associated with this file')

allensdk/test/brain_observatory/ecephys/test_copy_utility.py

@@ -10,7 +10,46 @@
 
 import allensdk.brain_observatory.ecephys.copy_utility.__main__ as cu
 from allensdk.brain_observatory.ecephys.copy_utility._schemas import (
-    SessionUploadInputSchema, SessionUploadOutputSchema)
+        SessionUploadInputSchema, SessionUploadOutputSchema)
+
+
+@pytest.mark.parametrize("already_exists", [True, False])
+def test_dst_dir_exists(already_exists, tmp_path, monkeypatch):
+    dst_dir = tmp_path / "new_directory"
+    src_file = tmp_path / "source.txt"
+    src_file.touch()
+
+    if already_exists:
+        dst_dir.mkdir()
+    dst_file = dst_dir / "destination.txt"
+    outj_path = tmp_path / "output.json"
+
+    args = {
+            "files": [
+                {
+                    "source": str(src_file),
+                    "destination": str(dst_file),
+                    "key": "something"}],
+            "output_json": str(outj_path)}
+
+    parser = argschema.ArgSchemaParser(
+        args,
+        schema_type=SessionUploadInputSchema,
+        output_schema_type=SessionUploadOutputSchema,
+        args=[]
+    )
+
+    def mock_copy_file(source, dest, use_rsync, make_parent_dirs, chmod=None):
+        parent = Path(dest).parent
+        if make_parent_dirs & (not parent.exists()):
+            parent.mkdir()
+        shutil.copy(source, dest)
+
+    monkeypatch.setattr(cu, "copy_file_entry", mock_copy_file)
+    output = cu.main(**parser.args)
+    parser.output(output, indent=2)
+    assert outj_path.exists()
+    assert Path(args["files"][0]["destination"]).exists()
 
 
 def test_hash_file(tmpdir_factory):

doc_template/index.rst

@@ -119,7 +119,12 @@ The Allen SDK provides Python code for accessing experimental metadata along wit
 See the `mouse connectivity section <connectivity.html>`_ for more details.
 
 
-What's New - 2.11.0
+What's New - 2.11.2
+-----------------------------------------------------------------------
+- Fixed mkdir error for non-existing ecephys upload directory
+
+
+What's New - 2.11.1
 -----------------------------------------------------------------------
 - Refactored the schema for the Ecephys copy utility to avoid raising an error when a previous output file already exists.