diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index dd351d81..ac0cef9f 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -1,12 +1,31 @@ name: "\U0001F41B Bug Report" description: Report a bug -title: "Bug: TITLE" -labels: ["bug"] +title: "Bug: (short bug description)" +labels: ["bug", "needs triage"] body: + - type: markdown + attributes: + value: | + Thank you for taking the time to fill out this bug report! + + ⚠️ If the bug that you are reporting is a security-related issue or security vulnerability, + then please do not create a report via this template. Instead please + notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) + or directly via email to [AWS Security](aws-security@amazon.com). + + - type: textarea + id: description + attributes: + label: Describe the bug + description: What is the problem? A clear and concise description of the bug. + validations: + required: true + - type: textarea id: expected_behaviour attributes: label: Expected Behaviour + description: What did you expect to happen? validations: required: true @@ -14,6 +33,7 @@ body: id: current_behaviour attributes: label: Current Behaviour + description: What actually happened? Please include as much detail as you can. validations: required: true @@ -21,13 +41,24 @@ body: id: reproduction_steps attributes: label: Reproduction Steps + description: | + Please provide as much detail as you can to help us understand how we can reproduce the bug. + Step by step instructions and self-contained code snippets are ideal. validations: required: true - type: textarea - id: code_snippet + id: environment attributes: - label: Code Snippet + label: Environment + description: Please provide information on the environment and software versions that you are using to reproduce the bug. + value: | + At minimum: + 1. Operating system: (e.g. Windows Server 2022; Amazon Linux 2023; etc.) + 2. Output of `python3 --version`: + 3. Output of `deadline --version`: + 4. If this is from a version installed by the Deadline Cloud Submitter installer, then what version of the submitter installer? + + Please share other details about your environment that you think might be relevant to reproducing the bug. validations: required: true - diff --git a/.github/ISSUE_TEMPLATE/doc.yml b/.github/ISSUE_TEMPLATE/doc.yml index 883623bb..7a95fad6 100644 --- a/.github/ISSUE_TEMPLATE/doc.yml +++ b/.github/ISSUE_TEMPLATE/doc.yml @@ -1,8 +1,8 @@ name: "📕 Documentation Issue" description: Issue in the documentation -title: "Docs: TITLE" -labels: ["documenation"] +title: "Docs: (short description of the issue)" +labels: ["documenation", "needs triage"] body: - type: textarea id: documentation_issue diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index ed7f9570..d2844682 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,17 +1,37 @@ name: "\U0001F680 Feature Request" description: Request a new feature -title: "Feature request: TITLE" -labels: ["feature"] +title: "Feature request: (short description of the feature)" +labels: ["feature", "needs triage"] body: - type: textarea - id: use_case + id: problem attributes: - label: Use Case + label: Describe the problem + description: | + Help us understand the problem that you are trying to solve, and why it is important to you. + Provide as much detail as you are able. validations: required: true + - type: textarea id: proposed_solution attributes: label: Proposed Solution + description: | + Describe your proposed feature that you see solving this problem for you. If you have a + full or partial prototype implementation then please open a draft pull request and link to + it here as well. + validations: + required: true + + - type: textarea + id: use_case + attributes: + label: Example Use Cases + description: | + Provide some sample code snippets or shell scripts that show how **you** would use this feature as + you have proposed it. validations: required: true + + diff --git a/.github/ISSUE_TEMPLATE/maintenance.yml b/.github/ISSUE_TEMPLATE/maintenance.yml index a0f98fff..f2440eb5 100644 --- a/.github/ISSUE_TEMPLATE/maintenance.yml +++ b/.github/ISSUE_TEMPLATE/maintenance.yml @@ -1,17 +1,19 @@ name: "🛠️ Maintenance" description: Some type of improvement -title: "Maintenance: TITLE" -labels: ["feature"] +title: "Maintenance: (short description of the issue)" +labels: ["feature", "needs triage"] body: - type: textarea id: description attributes: label: Description + description: Describe the improvement and why it is important to do. validations: required: true - type: textarea id: solution attributes: label: Solution + description: Provide any ideas you have for how the suggestion can be implemented. validations: required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 15d5333d..48b1113b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,5 @@ +Fixes: ** + ### What was the problem/requirement? (What/Why) ### What was the solution? (How) @@ -5,14 +7,32 @@ ### What is the impact of this change? ### How was this change tested? -- Have you run `hatch run test` ? -- Have you run `hatch run integ:test` ? See `DEVELOPMENT.md` on "Run integration tests" +See [DEVELOPMENT.md](https://github.com/aws-deadline/deadline-cloud/blob/mainline/DEVELOPMENT.md#testing) for information on running tests. + +- Have you run the unit tests? +- Have you run the integration tests? +- Have you made changes to the `download` or `asset_sync` modules? If so, then it is highly recommended + that you ensure that the docker-based unit tests pass. ### Was this change documented? +- Are relevant docstrings in the code base updated? +- Has the README.md been updated? If you modified CLI arguments, for instance. + ### Is this a breaking change? +A breaking change is one that modifies a public contract in a way that is not backwards compatible. See the +[Public Contracts](https://github.com/aws-deadline/deadline-cloud/blob/mainline/DEVELOPMENT.md#public-contracts) section +of the DEVELOPMENT.md for more information on the public contracts. + +If so, then please describe the changes that users of this package must make to update their scripts, or Python applications. + +### Does this change impact security? + +- Does the change need to be threat modeled? For example, does it create or modify files/directories that must only be readable by the process owner? + - If so, then please label this pull request with the "security" label. We'll work with you to analyze the threats. + ---- *By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.* \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 454d82f7..010697cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,62 +6,100 @@ documentation, we greatly value feedback and contributions from our community. Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. +Table of contents: + +* [Reporting Bugs/Feature Requests](#reporting-bugsfeature-requests) +* [Development](#development) + * [Finding contributions to work on](#finding-contributions-to-work-on) + * [Talk with us first](#talk-with-us-first) + * [Contributing via Pull Requests](#contributing-via-pull-requests) + * [Conventional Commits](#conventional-commits) +* [Licensing](#licensing) + ## Reporting Bugs/Feature Requests We welcome you to use the GitHub issue tracker to report bugs or suggest features. When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already -reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: - -* A reproducible test case or series of steps -* The version of our code being used -* Any modifications you've made relevant to the bug -* Anything unusual about your environment or deployment +reported the issue. Please try to include as much information as you can. ## Development -Please see [DEVELOPMENT.md](./DEVELOPMENT.md) for more information. +We welcome you to contribute features and bug fixes via a [pull request](https://help.github.com/articles/creating-a-pull-request/). +If you are new to contributing to GitHub repositories, then you may find the +[GitHub documentation on collaborating with the fork and pull model](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/getting-started/about-collaborative-development-models#fork-and-pull-model) +informative; this is the model that we follow. + +Please see [DEVELOPMENT.md](./DEVELOPMENT.md) for information about how to navigate this package's +code base and development practices. + +### Finding contributions to work on -## Contributing via Pull Requests +If you are not sure what you would like to contribute, then looking at the existing issues is a great way to find +something to contribute on. Looking at +[issues that have the "help wanted" or "good first issue" labels](https://github.com/aws-deadline/deadline-cloud/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22%2C%22help+wanted%22) +are a good place to start, but please dive into any issue that interests you whether it has those labels or not. + +### Talk with us first + +We ask that you please [open a feature request issue](https://github.com/aws-deadline/deadline-cloud/issues/new/choose) +(if one does not already exist) and talk with us before posting a pull request that contains a significant amount of work, +or one that proposes a change to a public interface such as to the interface of a publicly exported Python function or to +the command-line interfaces' commands or arguments. We want to make sure that your time and effort is respected by working +with you to design the change before you spend much of your time on it. If you want to create a draft pull request to show what +you are thinking and then talk with us, then that works with us as well. + +### Contributing via Pull Requests Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: -1. You are working against the latest source on the *main* branch. +1. You are working against the latest source on the *mainline* branch. 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. +4. Your pull request will be focused on a single change - it is easier for us to understand when a change is focused rather + than changing multiple things at once. To send us a pull request, please: 1. Fork the repository. -2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. -3. Ensure local tests pass. -4. Commit to your fork using clear commit messages. +2. Modify the source and add tests for your change; please focus on the specific change you are contributing. + If you also reformat all the code, it will be hard for us to focus on your change. + Please see [DEVELOPMENT.md](./DEVELOPMENT.md) for tips. +3. Ensure tests pass. Please see the [Testing](./DEVELOPMENT.md#testing) section for information on tests. +4. Commit to your fork using clear commit messages. Note that all AWS Deadline Cloud GitHub repositories require the use + of [conventional commit](#conventional-commits) syntax for the title of your commit. 5. Send us a pull request, answering any default questions in the pull request interface. 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. GitHub provides additional documentation on [forking a repository](https://help.github.com/articles/fork-a-repo/) and [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). -## Finding contributions to work on - -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. - +### Conventional commits -## Code of Conduct +The commits in this repository are all required to use [conventional commit syntax](https://www.conventionalcommits.org/en/v1.0.0/) +in their title to help us identify the kind of change that is being made, automatically generate the changelog, and +automatically identify next release version number. Only the first commit that deviates from mainline in your pull request +must adhere to this requirement. -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact -opensource-codeofconduct@amazon.com with any additional questions or comments. +We ask that you use these commit types in your commit titles: +* `feat` - When the pull request adds a new feature or functionality; +* `fix` - When the pull request is implementing a fix to a bug; +* `test` - When the pull request is only implementing an addition or change to tests or the testing infrastructure; +* `docs` - When the pull request is primarily implementing an addition or change to the package's documentation; +* `refactor` - When the pull request is implementing only a refactor of existing code; +* `ci` - When the pull request is implementing a change to the CI infrastructure of the packge; +* `chore` - When the pull request is a generic maintenance task. -## Security issue notifications +We also require that the type in your conventional commit title end in an exclaimation point (e.g. `feat!` or `fix!`) +if the pull request should be considered to be a breaking change in some way. Please also include a "BREAKING CHANGE" footer +in the description of your commit in this case ([example](https://www.conventionalcommits.org/en/v1.0.0/#commit-message-with-both--and-breaking-change-footer)). +Examples of breaking changes include any that implements a backwards-imcompatible change to a public Python interface, +the command-line interface, or the like. -We take all security reports seriously. When we receive such reports, we will -investigate and subsequently address any potential vulnerabilities as quickly -as possible. If you discover a potential security issue in this project, please -notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) -or directly via email to [AWS Security](aws-security@amazon.com). Please do not -create a public GitHub issue in this project. +If you need change a commit message, then please see the +[GitHub documentation on the topic](https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/changing-a-commit-message) +to guide you. ## Licensing diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d3967997..3913af73 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -2,100 +2,206 @@ This documentation provides guidance on developer workflows for working with the code in this repository. -## Code organization +Table of Contents: +* [Development Environment Setup](#development-environment-setup) +* [The Development Loop](#the-development-loop) +* [Code Organization](#code-organization) +* [Testing](#testing) + * [Writing tests](#writing-tests) + * [Unit tests](#unit-tests) + * [Integration tests](#integration-tests) +* [Things to Know](#things-to-know) + * [Public contracts](#public-contracts) + * [Qt and Calling AWS APIs](#qt-and-calling-aws-including-aws-deadline-cloud-apis) -This repository is split up into two main modules: -1. `src/client` -2. `src/job_attachments` +## Development Environment Setup -The `src/client` organization is laid out below. +To develop the Python code in this repository you will need: -For more information on job attachments, see [here](src/deadline/job_attachments/README.md). +1. Python 3.8 or higher. We recommend [mise](https://github.com/jdx/mise) if you would like to run more than one version + of Python on the same system. When running unit tests against all supported Python versions, for instance. +2. The [hatch](https://github.com/pypa/hatch) package installed (`pip install --upgrade hatch`) into your Python environment. -### `src/client/api` +You can develop on a Linux, MacOs, or Windows workstation, but you may find that some of the support scripting is specific to +Linux/MacOS workstations. -This submodule contains utilities to call boto3 in a standardized way -using an aws profile configured for AWS Deadline Cloud, helpers for working with the -AWS Deadline Cloud monitor login/logout, and objects representing AWS Deadline Cloud -resources. +If you are making changes to the Job Attachments files, then you will also need the following to be able to run the integration +tests: -### `src/client/cli` +1. A valid AWS Account +2. An AWS Deadline Cloud Farm and Queue. + * You can create these via AWS Deadline Cloud's AWS Console quick Farm create workflow. + The Queue's configuration must include a Job Attachments bucket. If used only for running these tests then the cost of + this infrastructure should be negligible, but do keep an eye on your costs and destroy the infrastructure (especially S3 buckets) + when you no longer need it. -This submodule contains entry points for the CLI applications provided -by the library. +## The Development Loop -### `src/client/config` +We have configured [hatch](https://github.com/pypa/hatch) commands to support a standard development loop. You can run the following +from any directory of this repository: -This submodule contains an interface to the machine-specific AWS Deadline Cloud -configuration, specifically settings stored in `~/.deadline/*` +* `hatch build` - To build the installable Python wheel and sdist packages into the `dist/` directory. +* `hatch run test` - To run the PyTest unit tests found in the `test/unit` directory. See [Testing](#testing). +* `hatch run all:test` - To run the PyTest unit tests against all available supported versions of Python. +* `hatch run integ:test` - To run the PyTest integration tests found in the `test/integ` directory. See [Testing](#testing). +* `hatch run lint` - To check that the package's formatting adheres to our standards. +* `hatch run fmt` - To automatically reformat all code to adhere to our formatting standards. +* `hatch shell` - Enter a shell environment where you can run the `deadline` command-line directly as it is implemented in your + checked-out local git repository. +* `hatch env prune` - Delete all of your isolated workspace [environments](https://hatch.pypa.io/1.12/environment/) + for this package. -### `src/client/ui` +If you are not sure about how to approach development for this package, then we suggest a development +process along the lines of the following as a starting point: -This submodule contains Qt GUIs, based on PySide(2/6), for common controls -and widgets used in interactive submitters, and to display the status -of various AWS Deadline Cloud resources. +1. Make your functional changes and make sure that they work. +2. Add unit tests for your changes and ensure that all unit tests pass. + Iteratively improve your implementation until all unit tests pass. (See [Unit tests](#unit-tests)) +3. Add integration tests for your changes if applicable. Ensure that all integration tests pass. + Iteratively improve your implementation until all integration and unit tests pass. (See [Integration tests](#integration-tests)) -### `src/client/job_bundle` +Once you are satisfied with your code, and all relevant tests pass, then run `hatch run fmt` to fix up the formatting of +your code and post your pull request. -This submodule contains code related to the history of job submissions -performed on the workstation. Its initial functionality is to create -job bundle directories in a standardized manner. +Note: Hatch uses [environments](https://hatch.pypa.io/1.12/environment/) to isolate the Python development workspace +for this package from your system or virtual environment Python. If your build/test run is not making sense, then +sometimes pruning (`hatch env prune`) all of these environments for the package can fix the issue. -# Build / Test / Release +## Code Organization -## Build the package. -``` -hatch build -``` +Please see [code organization](docs/code_organization.md). -## Run tests -``` -hatch run test -``` +## Testing -## Run integration tests -- Pre-requisites: - - Developer must have a valid AWS account. - - Developer must create a Farm and Queue. This can be done easily via Deadline Cloud's AWS Console quick Farm create workflow. Please copy the Farm resource ID for the next step. Please copy the Queue's Job Attachment S3 bucket name and Root Prefix for the next step. +The objective for the tests of this package are to act as regression tests to help identify unintended changes to +functionality in the package. As such, we strive to have high test coverage of the different behaviours/functionality +that the package contains. Code coverage metrics are not the goal, but rather are a guide to help identify places +where there may be gaps in testing coverage. -- Setup Environment Variables -```bash -export SERVICE_ACCOUNT_ID="your AWS Account Id" -export JOB_ATTACHMENTS_BUCKET="your Queue configured Job Attachments bucket" -export JA_TEST_ROOT_PREFIX="/your/path/root" -export FARM_ID="farm-{uuid}" -export AWS_DEFAULT_REGION="us-west-2" -``` +The tests for this package have two forms: -- Optional Environment Variables -```bash -export AWS_ENDPOINT_URL_DEADLINE="https://deadline.$AWS_DEFAULT_REGION.amazonaws.com" -export INTEG_TEST_JA_CROSS_ACCOUNT_BUCKET="Your regional S3 bucket in another account" -``` +1. Unit tests - Small tests that are narrowly focused on ensuring that function-level behavior + of the implementation behaves as it is expected to. These can always be run locally on your workstation + without requiring an AWS account. +2. Integration tests - Tests that ensure that the implementation behaves as expected when run in a real environment. + Ensuring that code properly interacts as expected with a real Amazon S3 bucket, for instance. -- Running the integration tests. -``` -hatch run integ:test -``` +### Writing Tests -- Notes: - - If the integration test is run in any region other than `us-west-2`, please set the environment variable `INTEG_TEST_JA_CROSS_ACCOUNT_BUCKET`. The bucket must be from another AWS account in the same region as the farm setup to run this test. -## Run linting -``` -hatch run lint -``` +If you want assistance developing tests, then please don't hesitate to open a draft pull request and ask for help. +We'll do our best to help you out and point you in the right direction. -## Run formating -``` -hatch run fmt -``` +Our tests are implemented using the [PyTest](https://docs.pytest.org/en/stable/) testing framework, +and unit tests generally make use of Python's [unittest.mock](https://docs.python.org/3.8/library/unittest.mock.html) +package to avoid runtime dependencies and narrowly focus tests on a specific aspect of the implementation. + +If you are not sure how to start writing tests, then we suggest looking at the existing tests +for the same or similar functions for inspiration (search for calls to the function within the `test/` +subdirectories). You will also find both the official [PyTest documentation](https://docs.pytest.org/en/stable/) +and [unitest.mock documentation](https://docs.python.org/3.8/library/unittest.mock.html) very informative (we do). + +### Unit Tests + +Unit tests are all located under the `test/unit` directory of this repository. If you are adding or modifying +functionality, then you will almost always want to be writing one or more unit tests to demonstrate that your +logic behaves as expected and that future changes do not accidentally break your change. -## Run tests for all supported Python versions. +#### Running Unit Tests + +You can run unit tests by running: + +* `hatch run test` - To run the unit tests with your default Python runtime. +* `hatch run all:test` - To run the unit tests with all of the supported Python runtime versions that you have installed. + +#### Running Docker-based Unit Tests + +Some of the unit tests in this package require a docker environment to run. These tests are marked with `@pytest.mark.docker`. +In order to run these tests, please run the `run_sudo_tests.sh` script located in the `scripts` directory. For detailed instructions, +please refer to [scripts/README.md](./scripts/README.md). + +If you make changes to the `download` or `asset_sync` modules, it's highly recommended to run and ensure these tests pass. + +### Integration Tests + +Integration tests are all located under the `test/integ` directory of this repository. You should consider +adding or modifying an integration test for any change that adds or modifies functionality that directly +interfaces with the local filesystem or an AWS service API. + +#### Running Integration Tests + +Our integration tests run using using infrastructure that is in your AWS Account. The identifiers for +these resources are communicated to the tests through environment variables that you must define before running +the tests. Define the following environment variables: + +```bash +# Replace with your AWS Account ID +export SERVICE_ACCOUNT_ID=000000000000 +# Replace with the region code where your AWS test resources are located (e.g. us-west-2) +export AWS_DEFAULT_REGION=xx-yyyy-nn +# Replace with the ID of your AWS Deadline Cloud Farm +export FARM_ID=farm-00112233445566778899aabbccddeeff +# Replace with the ID of your AWS Deadline Cloud Queue that is configured with a +# Job Attachments bucket. +export QUEUE_ID=queue-00112233445566778899aabbccddeeff + +export JOB_ATTACHMENTS_BUCKET=$( + aws deadline get-queue --farm-id $FARM_ID --queue-id $QUEUE_ID \ + --query 'jobAttachmentSettings.s3BucketName' | tr -d '"' +) +export JA_TEST_ROOT_PREFIX=$( + aws deadline get-queue --farm-id $FARM_ID --queue-id $QUEUE_ID \ + --query 'jobAttachmentSettings.rootPrefix' | tr -d '"' +) ``` -hatch run all:test + +Then you can run the integration tests with: + +```bash +hatch run integ:test ``` -## Qt and Calling AWS (including AWS Deadline Cloud) APIs +Notes: +* If you are not one of the AWS Deadline Cloud developers then you may see test failures in tests marked with + `pytest.mark.cross_account`. That's okay, just ignore them; they'll be tested with the required setup in our CI. +* If you are adding/changing code related to the Job Attachments' file-upload interactions with S3, then if you have a second + AWS account then we request that you also ensure that the tests marked with the `pytest.mark.cross_account` marker also pass. + If you don't have a second account, then don't worry about it. These tests will run in our CI. To run these tests: + 1. Create an S3 bucket in the same region as your testing resources but in your second AWS Account. + 2. Set the access policy of that S3 bucket to allow your first AWS Account to perform all operations on the bucket. Do + NOT open the bucket up to the world for reading/writing! + 3. `export INTEG_TEST_JA_CROSS_ACCOUNT_BUCKET=` + 4. Run the integration tests. +* AWS Developers note: If testing with a non-production deployment of AWS Deadline Cloud then you will have to +define the `AWS_ENDPOINT_URL_DEADLINE` environment variable to the non-production endpoint URL. For example, +production endpoints look like: `export AWS_ENDPOINT_URL_DEADLINE="https://deadline.$AWS_DEFAULT_REGION.amazonaws.com"` + +## Things to Know + +### Public Contracts + +The publicly consumable interfaces of this library and CLI are all considered to be public contracts. Meaning that any +change to them that is not backwards compatible is considered to be a breaking change. We strive to avoid making breaking +changes when possible, but accept that there are sometimes very good reasons for why a breaking change is necessary. + +The following are some heuristics to demonstrate how to think about breaking vs non-breaking changes in the public interface. + +For the command-line interface: +* Things like adding a non-required argument to a subcommand, or adding a new subcommand are not breaking changes. +* Renaming a subcommand or argument is a breaking change. +* Adding a new required subcommand argument is a breaking change. +* Changing a default value/behaviour is a breaking change. + +For the Python library interface: +* We follow the [PEP 8](https://peps.python.org/pep-0008/#descriptive-naming-styles) weak internal use indicator convention + and name all functions and modules that are internal/private with a leading underscore character. +* All functions and modules whose name does not begin with an underscore are part of the public contract for this package. +* Things like adding a non-required keyword argument to a function, or adding a new public function are not breaking changes. +* Things like renaming a keyword argument, or adding/removing a positional argument in a public function is a breaking change. +* Changing a default argument value is a breaking change. +* Changing the location that a file or directory is created should be considered to be a breaking change. These locations have a tendancy to become + de-facto parts of the public contract as users build automation that assumes these locations is unchanged. + +### Qt and Calling AWS (including AWS Deadline Cloud) APIs > TL;DR Never call an AWS API from the main Qt event loop. Always run it in a separate thread, > and use a Signal/Slot to send the result back to GUI widget that needs an update. The code @@ -185,9 +291,3 @@ class MyCustomWidget(QWidget): ``` -**We recommend you set up your runtimes via `mise`.** - -## Running Docker-based Unit Tests - -- Some of the unit tests in this package require a docker environment to run. These tests are marked with `@pytest.mark.docker`. In order to execute these tests, please run the `run_sudo_tests.sh` script located in the `scripts` directory. For detailed instructions, please refer to [scripts/README.md](./scripts/README.md). -- If you make changes to the `download` or `asset_sync` modules, it's highly recommended to run and ensure these tests pass. diff --git a/README.md b/README.md index a0b4afab..91088a22 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,25 @@ [![python](https://img.shields.io/pypi/pyversions/deadline.svg?style=flat)](https://pypi.python.org/pypi/deadline) [![license](https://img.shields.io/pypi/l/deadline.svg?style=flat)](https://github.com/aws-deadline/deadline/blob/mainline/LICENSE) +AWS Deadline Cloud client is a multi-purpose python library and command line tool for interacting with and submitting +[Open Job Description (OpenJD)][openjd] jobs to [AWS Deadline Cloud][deadline-cloud]. -AWS Deadline Cloud client is a multi-purpose python library and command line tool for interacting with and submitting [Open Job Description (OpenJD)][openjd] jobs to [AWS Deadline Cloud][deadline-cloud]. +To support building workflows on top of AWS Deadline Cloud it implements its own user interaction, job creation, file upload/download, and other useful +helpers around the service's api. It can function as a pipeline tool, a standalone gui application, or even be embedded within other applications' runtimes. -To support building workflows on top of AWS Deadline Cloud it implements its own user interaction, job creation, file upload/download, and other useful helpers around the service's api. It can function as a pipeline tool, a standalone gui application, or even be embedded within other applications' runtimes. +Notable features include: +* A command-line interface with subcommands for querying your AWS Deadline Cloud resources, and submitting jobs to your AWS Deadline Cloud Farm. +* A library of functions that implement AWS Deadline Cloud's Job Attachments functionality. +* A library of functions for creating a job submission UI within any content creation tool that supports Python 3.8+ based plugins and + the Qt GUI framework. [cas]: https://en.wikipedia.org/wiki/Content-addressable_storage [deadline-cloud]: https://docs.aws.amazon.com/deadline-cloud/latest/userguide/what-is-deadline-cloud.html [deadline-cloud-monitor]: https://docs.aws.amazon.com/deadline-cloud/latest/userguide/working-with-deadline-monitor.html [deadline-cloud-samples]: https://github.com/aws-deadline/deadline-cloud-samples [deadline-jobs]: https://docs.aws.amazon.com/deadline-cloud/latest/userguide/deadline-cloud-jobs.html -[job-attachments]: https://github.com/aws-deadline/deadline-cloud/blob/mainline/src/deadline/job_attachments/README.md +[job-attachments]: https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-attachments.html +[job-bundles]: https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle.html [openjd]: https://github.com/OpenJobDescription/openjd-specifications/wiki ## Compatibility @@ -24,6 +32,23 @@ This library requires: 1. Python 3.8 or higher; and 2. Linux, Windows, or macOS operating system. +## Versioning + +This package's version follows [Semantic Versioning 2.0](https://semver.org/), but is still considered to be in its +initial development, thus backwards incompatible versions are denoted by minor version bumps. To help illustrate how +versions will increment during this initial development stage, they are described below: + +1. The MAJOR version is currently 0, indicating initial development. +2. The MINOR version is currently incremented when backwards incompatible changes are introduced to the public API. +3. The PATCH version is currently incremented when bug fixes or backwards compatible changes are introduced to the public API. + +## Contributing + +We welcome all contributions. Please see [CONTRIBUTING.md](https://github.com/aws-deadline/deadline-cloud/blob/mainline/CONTRIBUTING.md) +for guidance on how to contribute. Please report issues like bugs, inaccurate or confusing information, and so on +and make feature requests in the [issue tracker](https://github.com/aws-deadline/deadline-cloud/issues). We encourage +code contributions in the form of [pull requests](https://github.com/aws-deadline/deadline-cloud/pulls). + ## Getting Started The AWS Deadline Cloud client can be installed by the standard python packaging mechanisms: @@ -54,12 +79,19 @@ api.list_farms() ## Job attachments -Job attachments enable you to transfer files between your workstations and AWS Deadline Cloud, by using Amazon S3 buckets as [content-addressed storage][cas] in your AWS account. +Job attachments enable you to transfer files between your workstations and AWS Deadline Cloud, by using Amazon S3 buckets as +[content-addressed storage][cas] in your AWS account. The use of a content-addressed storage means that a file will never need +to be uploaded again once it has been uploaded once. See [job attachments][job-attachments] for a more in-depth look at how files are uploaded, stored, and retrieved. ## Job bundles +A job bundle is one of the tools that you can use to define jobs for AWS Deadline Cloud. They group an [Open Job Description](openjd) template with +additional information such as files and directories that your jobs use with job attachments. You can use this package's command-line interface and/or +its Python interface to use a job bundle to submit jobs for a queue to run. Please see the [Job Bundles](job-bundles) +section of the AWS Deadline Cloud Developer Guide for detailed information on job bundles. + At minimum a job bundle is a folder that contains an [OpenJD][openjd] template as a file named `template.json` or `template.yaml`, however it can optionally include: 1. an `asset_references.yaml` - lists file inputs and outputs, 2. a `parameter_values.yaml` - contains the selected values for the job template's parameters, @@ -79,8 +111,6 @@ $ deadline bundle gui-submit --browse On submission a job bundle will be created in the job history directory (default: `~/.deadline/job_history`). -For more information on jobs and job bundles, see [AWS Deadline Cloud jobs][deadline-jobs]. - ## Configuration You can see the current configuration by running: @@ -126,34 +156,30 @@ and removing them by logging out: $ deadline auth logout ``` -## Versioning - -This package's version follows [Semantic Versioning 2.0](https://semver.org/), but is still considered to be in its -initial development, thus backwards incompatible versions are denoted by minor version bumps. To help illustrate how -versions will increment during this initial development stage, they are described below: - -1. The MAJOR version is currently 0, indicating initial development. -2. The MINOR version is currently incremented when backwards incompatible changes are introduced to the public API. -3. The PATCH version is currently incremented when bug fixes or backwards compatible changes are introduced to the public API. - -## Contributing +## Code of Conduct -See [CONTRIBUTING](https://github.com/aws-deadline/deadline-cloud/blob/mainline/CONTRIBUTING.md) -for information on reporting issues, requesting features, and developer information. +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. -## Security +## Security issue notifications -See [security issue notifications](https://github.com/aws-deadline/deadline-cloud/blob/release/CONTRIBUTING.md#security-issue-notifications) for more information. +We take all security reports seriously. When we receive such reports, we will +investigate and subsequently address any potential vulnerabilities as quickly +as possible. If you discover a potential security issue in this project, please +notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) +or directly via email to [AWS Security](aws-security@amazon.com). Please do not +create a public GitHub issue in this project. ## Telemetry See [telemetry](https://github.com/aws-deadline/deadline-cloud/blob/release/docs/telemetry.md) for more information. -## Optional third party dependencies - GUI - -N.B.: Although this repository is released under the Apache-2.0 license, its optional GUI feature -uses the third party Qt and PySide projects. The Qt and PySide projects' licensing includes the LGPL-3.0 license. - ## License This project is licensed under the Apache-2.0 License. + +### Optional third party dependencies - GUI + +N.B.: Although this repository is released under the Apache-2.0 license, its optional GUI feature +uses the third party Qt and PySide projects. The Qt and PySide projects' licensing includes the LGPL-3.0 license. diff --git a/docs/code_organization.md b/docs/code_organization.md new file mode 100644 index 00000000..a3015fc6 --- /dev/null +++ b/docs/code_organization.md @@ -0,0 +1,38 @@ +# Code organization + +This repository is split up into two main modules: +1. `src/client` +2. `src/job_attachments` + +The `src/client` organization is laid out below. + +For more information on job attachments, see [here](src/deadline/job_attachments/README.md). + +### `src/client/api` + +This submodule contains utilities to call boto3 in a standardized way +using an aws profile configured for AWS Deadline Cloud, helpers for working with the +AWS Deadline Cloud monitor login/logout, and objects representing AWS Deadline Cloud +resources. + +### `src/client/cli` + +This submodule contains entry points for the CLI applications provided +by the library. + +### `src/client/config` + +This submodule contains an interface to the machine-specific AWS Deadline Cloud +configuration, specifically settings stored in `~/.deadline/*` + +### `src/client/ui` + +This submodule contains Qt GUIs, based on PySide(2/6), for common controls +and widgets used in interactive submitters, and to display the status +of various AWS Deadline Cloud resources. + +### `src/client/job_bundle` + +This submodule contains code related to the history of job submissions +performed on the workstation. Its initial functionality is to create +job bundle directories in a standardized manner. \ No newline at end of file diff --git a/test/integ/deadline_job_attachments/test_job_attachments.py b/test/integ/deadline_job_attachments/test_job_attachments.py index 7a3a6fd8..94f82e6f 100644 --- a/test/integ/deadline_job_attachments/test_job_attachments.py +++ b/test/integ/deadline_job_attachments/test_job_attachments.py @@ -1284,6 +1284,7 @@ def on_downloading_files(*args, **kwargs): assert not any(Path(session_dir).iterdir()) +@pytest.mark.cross_account @pytest.mark.integ def test_upload_bucket_wrong_account(external_bucket: str, job_attachment_test: JobAttachmentTest): """ @@ -1307,7 +1308,11 @@ def test_upload_bucket_wrong_account(external_bucket: str, job_attachment_test: # WHEN with pytest.raises( - JobAttachmentsS3ClientError, match=".*when calling the PutObject operation: Access Denied" + # Note: This error is raised in this case when the s3:PutObject operation is denied + # due to the ExpectedBucketOwner check on our s3 operation. If the bucket is in the expected + # account, then the error is a different access denied error. + JobAttachmentsS3ClientError, + match=".*when calling the PutObject operation: Access Denied", ): # The attempt to upload the asset manifest should be blocked. upload_group = asset_manager.prepare_paths_for_upload( @@ -1329,6 +1334,7 @@ def test_upload_bucket_wrong_account(external_bucket: str, job_attachment_test: ) +@pytest.mark.cross_account @pytest.mark.integ def test_sync_inputs_bucket_wrong_account( external_bucket: str, @@ -1377,6 +1383,7 @@ def on_downloading_files(*args, **kwargs): ) +@pytest.mark.cross_account @pytest.mark.integ def test_sync_outputs_bucket_wrong_account( job_attachment_test: JobAttachmentTest, @@ -1385,6 +1392,8 @@ def test_sync_outputs_bucket_wrong_account( ) -> None: """ Test that if trying to sync outputs to a bucket that isn't in the farm's AWS account, the correct error is thrown. + This is ensuring that the S3 file upload is passing the ExpectedBucketOwner property and verifying that the returned + error is what we expect when using that property (rather than just plain not having access to the bucket). """ # IF job_attachment_settings = JobAttachmentS3Settings( @@ -1458,6 +1467,7 @@ def test_sync_outputs_bucket_wrong_account( ) +@pytest.mark.cross_account @pytest.mark.integ def test_download_outputs_bucket_wrong_account( job_attachment_test: JobAttachmentTest,