refactor!: General cleanup and polish

BREAKING-CHANGE: Some of the renames (errors module to exceptions, aws module to _aws) are not backwards compatible. - Rename internal functions to start with '_' to follow Python's conventions. - Refine the wording of docstrings and exception messages in the login/lout functionality. Use DeadlineOperationError to prevent stack traces for errors with error messages we've curated. - Tidy up the main() function to use the one generated by click directly instead of wrapping it. - Mark the bealine-dev-gui command as experimental. - Simplify hatch_version_hook.py to hatch_custom_hook.py the same way as in deadline-cloud-for-maya. - Change the default log level to WARNING instead of INFO. Remove the code that overrode the log level when importing deadline.client.ui, we do not want to mess with settings that clients of the library determine. - Update the deadline dev gui entry point to use the log level setting. - Remove the dataclasses-json dependency. This transitively also depends on marshmallow mypy-extensions packaging typing-extensions, and typing-inspect. Because deadline-cloud is imported into customer code contexts that we have little control over, every additional dependency adds the risk of conflicts with choices that customers have made. Therefore each dependency must provide enough benefit to justify this risk, and dataclasses-json does not, as we can implement to_dict and from_dict methods directly for the small number of dataclasses that need this. - Remove the old manifest version v2022_06_06. Update all the conditional code about v2023_03_03 to be the default. Clean up some names like Path -> ManifestPath to avoid confusion with pathlib. - Remove the src/deadline_job_attachments shim that was used temporarily while transitioning to a namespace package. - Remove the map_source_path_to_dest_path function, as it wasn't performing a needed path transformation. - Rename job_attachments.errors to job_attachments.exceptions to be consistent with the exceptions module in the client lib. - Clean up examples/submit_job.py to not depend on any of the interfaces defined as internal. - Combine the duplicated hashing & uploading summary output into the SummaryStatistics __str__ method. - Remove the get_rez_environment function, as we are moving it fully to a Queue Environment. This also means the OPENJDToken class in utils is no longer used, so also remove that. Signed-off-by: Mark Wiebe <[email protected]>
godobyte · Sep 11, 2023 · 608d225 · 608d225
1 parent f1b2d61
commit 608d225
Show file tree

Hide file tree

Showing 91 changed files with 801 additions and 1,978 deletions.
diff --git a/THIRD_PARTY_LICENSES b/THIRD_PARTY_LICENSES
diff --git a/examples/download_cancel_test.py b/examples/download_cancel_test.py
@@ -8,9 +8,9 @@
 from threading import Thread
 
 from deadline.job_attachments.asset_sync import AssetSync
-from deadline.job_attachments.aws.deadline import get_job, get_queue
+from deadline.job_attachments._aws.deadline import get_job, get_queue
 from deadline.job_attachments.download import OutputDownloader
-from deadline.job_attachments.errors import AssetSyncCancelledError
+from deadline.job_attachments.exceptions import AssetSyncCancelledError
 
 """
 A testing script to simulate cancellation of (1) syncing inputs, and (2) downloading outputs.

diff --git a/examples/download_output.py b/examples/download_output.py
@@ -6,7 +6,7 @@
 import sys
 import time
 
-from deadline.job_attachments.aws.deadline import get_queue
+from deadline.job_attachments._aws.deadline import get_queue
 from deadline.job_attachments.download import OutputDownloader
 
 """

diff --git a/examples/submit_job.py b/examples/submit_job.py
@@ -11,38 +11,35 @@
 """
 
 import argparse
-import json
 import pprint
 import time
 from pathlib import Path
 
 import boto3
-from openjd.model.template import SchemaVersion, TemplateModelRegistry
 
-from deadline.job_attachments.aws.deadline import get_queue
 from deadline.job_attachments.upload import S3AssetManager
-from deadline.job_attachments.utils import (
-    get_deadline_formatted_os,
-    get_unique_dest_dir_name,
-    map_source_path_to_dest_path,
-)
+from deadline.job_attachments.models import JobAttachmentS3Settings
 
 
-def process_job_attachments(farm_id, queue_id, inputs, outputDir):
+def process_job_attachments(farm_id, queue_id, inputs, outputDir, deadline_client):
     """
     Uploads all of the input files to the Job Attachments S3 bucket associated with
     the Deadline Queue, returning Attachment Settings to be associated with a Deadline Job.
     """
 
     print("Getting queue information...")
     start = time.perf_counter()
-    queue = get_queue(farm_id=farm_id, queue_id=queue_id)
+    queue = deadline_client.get_queue(farmId=farm_id, queueId=queue_id)
     total = time.perf_counter() - start
     print(f"Finished getting queue information after {total} seconds.\n")
 
     print(f"Processing {len(inputs)} job attachments...")
     start = time.perf_counter()
-    asset_manager = S3AssetManager(job_attachment_settings=queue.jobAttachmentSettings)
+    asset_manager = S3AssetManager(
+        farm_id=farm_id,
+        queue_id=queue_id,
+        job_attachment_settings=JobAttachmentS3Settings(**queue["jobAttachmentSettings"]),
+    )
     (_, manifests) = asset_manager.hash_assets_and_create_manifest(inputs, [outputDir])
     (_, attachments) = asset_manager.upload_assets(manifests)
     attachments = attachments.to_dict()
@@ -53,96 +50,61 @@ def process_job_attachments(farm_id, queue_id, inputs, outputDir):
     return attachments
 
 
-def create_job_template(asset_root, outputDir):
-    """
-    Creates a Job Template that defines a Task that counts the number of files and total size,
+JOB_TEMPLATE = """specificationVersion: '2022-09-01'
+name: SubmitJobExample
+description: >
+    A Job that counts the number of files and total size,
     and also creates a default output file.
-    """
-
-    # Since we're hardcoding paths in this job template, we need to determine what
-    # the session directory will be, using the same code as the Job Attachments lib.
-    session_dir = get_unique_dest_dir_name(asset_root)
-    # Map everything to linux for now since Workers only run on Linux
-    # The only reason we need to do this mapping is because we're hardcoding paths
-    # in the job template, so we have to assume what OS the Worker will have.
-    root_path = map_source_path_to_dest_path(get_deadline_formatted_os(), "linux", asset_root)
-    rel_output = map_source_path_to_dest_path(
-        get_deadline_formatted_os(), "linux", outputDir
-    ).relative_to(root_path)
-
-    # Create Job Template
-    template_model = TemplateModelRegistry.get_template_model(version=SchemaVersion.v2022_05_01)
-
-    script_start = ["#!/bin/env bash", "set -ex"]
-
-    run = template_model.InlineTextAttachmentEntity(
-        version=template_model.schema_version,
-        name="run",
-        data="\n".join(
-            [
-                *script_start,
-                "echo 'Confirming that inputs were downloaded to the correct location'",
-                f"echo 'Total number of inputs' && find {{{{ Builtin.SessionDirectory }}}}/{session_dir}/ -type f | wc -l",
-                f"echo 'Total file size' && du -hs {{{{ Builtin.SessionDirectory }}}}/{session_dir}/",
-                "echo 'Creating the expected output directory and output file'",
-                f"mkdir -p {{{{ Builtin.SessionDirectory }}}}/{session_dir}/{rel_output}",
-                f"echo 'This is test output' > {{{{ Builtin.SessionDirectory }}}}/{session_dir}/{rel_output}/output.txt",
-            ]
-        ),
-        is_runnable=True,
-    )
-
-    step = template_model.StepTemplateEntity(
-        version=template_model.schema_version,
-        name="custom-step",
-        script=template_model.StepScriptEntity(
-            version=template_model.schema_version,
-            actions=template_model.StepActionsEntity(
-                version=template_model.schema_version,
-                run=template_model.ActionEntity(
-                    version=template_model.schema_version,
-                    command=f"{{{{ {template_model.TASK_ATTACHMENT_NAMESPACE}.run.Path }}}}",
-                ),
-            ),
-            attachments=[
-                run,
-            ],
-        ),
-        parameter_space=template_model.StepParameterSpaceEntity(
-            version=template_model.schema_version,
-            parameters=[
-                template_model.TaskParameterEntity(
-                    version=template_model.schema_version,
-                    name="frame",
-                    range_list=["0"],
-                )
-            ],
-        ),
-    )
-
-    return template_model.JobTemplateEntity(
-        version=template_model.schema_version, name="custom-job", steps=[step]
-    )
+parameters:
+  - name: DataDir
+    type: PATH
+    objectType: DIRECTORY
+    dataFlow: INOUT
+  - name: RelOutput
+    type: PATH
+steps:
+  - name: layerDefaultFrames
+    script:
+        actions:
+            onRun:
+                command: '{{Task.File.Run}}'
+        embeddedFiles:
+          - name: Run
+            filename: count-files.sh
+            type: TEXT
+            runnable: true
+            data: |
+                #!/bin/env bash
+
+                set -euo pipefail
+                echo 'Confirming that inputs were downloaded to the correct location'",
+                echo 'Total number of inputs' && find {{Param.DataDir}} -type f | wc -l",
+                echo 'Total file size' && du -hs {{Param.DataDir}}",
+                echo 'Creating the expected output directory and output file'",
+                mkdir -p {{Param.DataDir}}/{{Param.RelOutput}}",
+                echo 'This is test output' > {{Param.DataDir}}/{{Param.RelOutput}}/output.txt",
+"""
 
 
-def submit_custom_job(farm_id, queue_id, job_template, attachment_settings):
+def submit_custom_job(
+    farm_id, queue_id, job_template, attachment_settings, parameters, deadline_client
+):
     """
     Submits a Job defined in the Job Template to the given Queue, adding the givent Attachment Settings
     to the Job definition.
     """
 
-    template_json = json.dumps(job_template.encode())
-
     # Submit the Job
     print("Submitting the job...")
     start = time.perf_counter()
-    deadline = boto3.client("deadline", region_name="us-west-2")
-    response = deadline.create_job(
+    response = deadline_client.create_job(
         farmId=farm_id,
         queueId=queue_id,
-        template=template_json,
-        templateType="JSON",
-        attachmentSettings=attachment_settings if attachment_settings else None,
+        template=job_template,
+        templateType="YAML",
+        attachments=attachment_settings if attachment_settings else None,
+        parameters=parameters,
+        priority=50,
     )
     total = time.perf_counter() - start
     print(f"Submitted Job Template after {total} seconds:")
@@ -200,10 +162,26 @@ def submit_custom_job(farm_id, queue_id, job_template, attachment_settings):
         else:
             inputs.append(str(file_path))
 
-    attachments = process_job_attachments(args.farm_id, args.queue_id, inputs, args.output_dir)
+    deadline_client = boto3.client(
+        "deadline",
+        region_name="us-west-2",
+        endpoint_url="https://gamma.us-west-2.bealine.creative-tools.aws.dev",
+    )
+
+    attachments = process_job_attachments(
+        args.farm_id, args.queue_id, inputs, args.output_dir, deadline_client
+    )
 
     if not args.assets_only:
-        template = create_job_template(attachments["manifests"][0]["rootPath"], args.output_dir)
-        submit_custom_job(args.farm_id, args.queue_id, template, attachments)
+        root_dir = attachments["manifests"][0]["rootPath"]
+        rel_output = str(Path(args.output_dir).relative_to(root_dir))
+        submit_custom_job(
+            args.farm_id,
+            args.queue_id,
+            JOB_TEMPLATE,
+            attachments,
+            {"DataDir": {"path": root_dir}, "RelOutput": {"path": rel_output}},
+            deadline_client,
+        )
 
     print(f"\nTotal submit runtime: {time.perf_counter() - start_time}")
diff --git a/examples/sync_inputs_with_step_deps.py b/examples/sync_inputs_with_step_deps.py
@@ -8,7 +8,7 @@
 import time
 
 from deadline.job_attachments.asset_sync import AssetSync
-from deadline.job_attachments.aws.deadline import get_job, get_queue
+from deadline.job_attachments._aws.deadline import get_job, get_queue
 
 """
 A script to manually test that input syncing is functioning well in scenarios where

diff --git a/examples/upload_cancel_test.py b/examples/upload_cancel_test.py
@@ -7,8 +7,8 @@
 import time
 from threading import Thread
 
-from deadline.job_attachments.aws.deadline import get_queue
-from deadline.job_attachments.errors import AssetSyncCancelledError
+from deadline.job_attachments._aws.deadline import get_queue
+from deadline.job_attachments.exceptions import AssetSyncCancelledError
 from deadline.job_attachments.upload import S3AssetManager
 
 NUM_SMALL_FILES = 0

diff --git a/examples/upload_scale_test.py b/examples/upload_scale_test.py
@@ -7,7 +7,7 @@
 import sys
 import time
 
-from deadline.job_attachments.aws.deadline import get_queue
+from deadline.job_attachments._aws.deadline import get_queue
 from deadline.job_attachments.download import download_files_from_manifests, get_manifest_from_s3
 from deadline.job_attachments.upload import S3AssetManager
 

diff --git a/hatch_custom_hook.py b/hatch_custom_hook.py
@@ -0,0 +1,49 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+from __future__ import annotations
+
+import os
+import shutil
+
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+from typing import Any
+
+
+class HatchCustomBuildHook(BuildHookInterface):
+    """
+    This class implements Hatch's [custom build hook] (https://hatch.pypa.io/1.6/plugins/build-hook/custom/)
+    for a copy_version_py operation that copies the _version.py file generated by the hatch-vcs build hook into
+    specified destination directories. See the `[[tool.hatch.build.hooks.custom]]` section in `pyproject.toml`.
+    """
+
+    def _validate_config(self):
+        if sorted(self.config) != ["copy_version_py", "path"] or list(
+            self.config["copy_version_py"]
+        ) != ["destinations"]:
+            raise RuntimeError(
+                "Configuration of the custom build hook must be like { 'copy_version_py': {'destinations': ['path1', ...]}}."
+                + f" Received:\n{self.config}"
+            )
+
+    def initialize(self, version: str, build_data: dict[str, Any]) -> None:
+        self._validate_config()
+
+        for destination in self.config["copy_version_py"]["destinations"]:
+            print(f"Copying _version.py to {destination}")
+            shutil.copy(
+                os.path.join(self.root, "_version.py"),
+                os.path.join(self.root, destination),
+            )
+
+    def clean(self, versions: list[str]) -> None:
+        self._validate_config()
+
+        cleaned_count = 0
+        for destination in self.config["copy_version_py"]["destinations"]:
+            print(f"Cleaning _version.py from {destination}")
+            clean_path = os.path.join(self.root, destination, "_version.py")
+            try:
+                os.remove(clean_path)
+                cleaned_count += 1
+            except FileNotFoundError:
+                pass
+        print(f"Cleaned {cleaned_count} items")