diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..bc281f4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,20 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +charset = utf-8 +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true +max_line_length = 120 + +[{*.py,*.yml,*.toml}] +indent_size = 4 + +[*.sh] +switch_case_indent = true + +# Ignore the entire ".venv" directory. +[.venv/**] +ignore = true diff --git a/.gitignore b/.gitignore index 86adbeb..497184a 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,4 @@ tutorial/*.html .idea vizdoom.ini -lib/ \ No newline at end of file +lib/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4bd6cd..bb3e575 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,70 @@ stages: - - lint + - lint + - build + - publish licenses_checker: - stage: lint - image: registry.gitlab.com/ai-r/cogment/license-checker:latest - script: - - license-checker + stage: lint + image: registry.gitlab.com/ai-r/cogment/license-checker:latest + script: + - license-checker +shellcheck: + image: koalaman/shellcheck-alpine:stable + stage: lint + before_script: + - shellcheck --version + script: + - shellcheck $(find . -name '*.sh' | xargs) + +shfmt: + image: mvdan/shfmt:v3.7.0-alpine + stage: lint + before_script: + - shfmt --version + script: + - shfmt -d . + +.base: + image: python:3.10 + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + before_script: + - mkdir -p ${PIP_CACHE_DIR} + - python -m venv .venv + - source .venv/bin/activate + - pip install -r requirements.txt + - pip install -e . + cache: + - paths: + - .cache/pip + - key: + files: + - requirements.txt + - setup.cfg + paths: + - ".venv" + +build_sdist: + extends: .base + stage: build + script: + - echo "to be implemented" + # TODO + # - python -m build + # - twine check dist/* + artifacts: + expire_in: 1 week + paths: + - dist/*.tar.gz + - dist/*.whl + +publish_to_pypi: + extends: .base + stage: publish + needs: + - build_sdist + script: + - python -m twine upload dist/* --non-interactive -u $PYPI_USERNAME -p $PYPI_PASSWORD + rules: + - if: $CI_COMMIT_TAG diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..166ff7f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.1.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## Unreleased + +### Added + +- Initial release diff --git a/README.md b/README.md index 66398af..25a6d28 100644 --- a/README.md +++ b/README.md @@ -95,3 +95,21 @@ Terminology: - Model: a relatively raw PyTorch (or other?) model, inheriting from `nn.Module` - Agent: a model wrapped in some utility class to interact with np arrays - Actor: a cogment service that may involve models and/or actors + +## Release process + +People having maintainers rights of the repository can follow these steps to release a version **MAJOR.MINOR.PATCH**. The versioning scheme follows [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +1. Run `./scripts/create_release_branch.sh MAJOR.MINOR.PATCH`, this will automatically: + - update the version of the package, in `cogment_lab/version.py`, + - create a release branch with this changes at `release/vMAJOR.MINOR.PATCH` and push it. +2. On the release branch: + - Make sure the changelog, at `CHANGELOG.md`, reflects the changes since the last release, + - Fix any issue, making sure that te build passes on CI, + - Commit and push any changes. +3. Run `./scripts/tag_release.sh MAJOR.MINOR.PATCH`, this will automatically: + - create the specific version section in the changelog and push it to the release branch, + - merge the release branch in `main`, + - create the release tag and, + - update the `develop` to match the latest release. +4. The CI will automatically publish the package to PyPI. diff --git a/bin/docker_entrypoint b/bin/docker_entrypoint index a5fe2c9..553c443 100755 --- a/bin/docker_entrypoint +++ b/bin/docker_entrypoint @@ -11,16 +11,16 @@ export DISPLAY=:0 display=0 file="/tmp/.X11-unix/X$display" for i in $(seq 1 10); do - if [ -e "$file" ]; then - break - fi + if [ -e "$file" ]; then + break + fi - echo "Waiting for $file to be created (try $i/10)" - sleep "$i" + echo "Waiting for $file to be created (try $i/10)" + sleep "$i" done if ! [ -e "$file" ]; then - echo "Timing out: $file was not created" - exit 1 + echo "Timing out: $file was not created" + exit 1 fi exec "$@" diff --git a/cogment_lab/__init__.py b/cogment_lab/__init__.py index c0d2dc3..6aa5971 100644 --- a/cogment_lab/__init__.py +++ b/cogment_lab/__init__.py @@ -13,14 +13,16 @@ # limitations under the License. from __future__ import annotations + import logging -from cogment_lab.process_manager import Cogment from cogment.utils import logger -logger.addHandler(logging.NullHandler()) +from cogment_lab.process_manager import Cogment +from cogment_lab.version import __version__ + -__version__ = "0.0.1" +logger.addHandler(logging.NullHandler()) __all__ = ["Cogment"] diff --git a/cogment_lab/version.py b/cogment_lab/version.py new file mode 100644 index 0000000..87ffada --- /dev/null +++ b/cogment_lab/version.py @@ -0,0 +1,15 @@ +# Copyright 2024 AI Redefined Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.1.0" diff --git a/pyproject.toml b/pyproject.toml index 99fab85..90e961a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,19 +19,17 @@ dependencies = [ "numpy", "opencv-python>=4.8", "fastapi>=0.105", - "pillow>=9.0" + "pillow>=9.0", ] [project.scripts] cogmentlab = "cogment_lab.cli.cli:main" [tool.hatch.version] -path = "cogment_lab/__init__.py" +path = "cogment_lab/version.py" [tool.hatch.build.targets.sdist] -include = [ - "/cogment_lab", -] +include = ["/cogment_lab"] # Package ###################################################################### @@ -56,16 +54,8 @@ dev = [ "jupyter>=1.0.0", "jupyterlab>=3.5.3", ] -coltra = [ - "coltra>=0.2.1", - "torch", - "wandb>=0.13.9", -] -grid2op = [ - "Grid2Op==1.9.6", - "lightsim2grid", - "numba==0.56.4", -] +coltra = ["coltra>=0.2.1", "torch", "wandb>=0.13.9"] +grid2op = ["Grid2Op==1.9.6", "lightsim2grid", "numba==0.56.4"] [project.urls] Homepage = "https://cogment.ai/lab" @@ -80,9 +70,10 @@ include-package-data = true include = ["cogment_lab", "cogment_lab.*"] [tool.setuptools.package-data] -cogment_lab = [ - "py.typed", -] +cogment_lab = ["py.typed"] + +[tool.setuptools.dynamic.version] +attr = "cogment_lab.version.__version__" # Linters and Test tools ####################################################### @@ -121,8 +112,8 @@ reportInvalidTypeVarUse = "none" # reportUnknownParameterType = "warning" # -> raises 1327 warnings # reportUnknownVariableType = "warning" # -> raises 2585 warnings # reportUnknownArgumentType = "warning" # -> raises 2104 warnings -reportGeneralTypeIssues = "none" # -> commented out raises 489 errors -reportUntypedFunctionDecorator = "none" # -> pytest.mark.parameterize issues +reportGeneralTypeIssues = "none" # -> commented out raises 489 errors +reportUntypedFunctionDecorator = "none" # -> pytest.mark.parameterize issues reportPrivateUsage = "warning" reportUnboundVariable = "warning" @@ -156,7 +147,7 @@ exclude = [ "node_modules", "venv", "scripts", - "cogment_lab/generated" + "cogment_lab/generated", ] # Same as Black. @@ -189,4 +180,4 @@ indent-style = "space" skip-magic-trailing-comma = false # Like Black, automatically detect the appropriate line ending. -line-ending = "auto" \ No newline at end of file +line-ending = "auto" diff --git a/scripts/commons.sh b/scripts/commons.sh new file mode 100644 index 0000000..5cf3e97 --- /dev/null +++ b/scripts/commons.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +#### +# Helper functions & variables +#### + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export ROOT_DIR + +GIT_REMOTE="origin" +export GIT_REMOTE + +# Let's check if we have GNU sed or BSD sed +if sed --help >/dev/null 2>&1; then + # There is a '--help' option, it is GNU BSD + SED_NL="\\n" +else + SED_NL="\\ +" +fi +export SED_NL + +# Generic functions + +## join_by +## Examples: +## $ join_by "-delimiter-" "a" "b" "c" +## "a-delimiter-b-delimiter-c" +function join_by() { + local delimiter=$1 + shift + local strings=$1 + shift + printf %s "${strings}" "${@/#/$delimiter}" +} + +## array_contains +## Examples: +## $ array_contains "foo" "bar" "foobaz" +## 1 +## +## $ array_contains "foo" "bar" "foo" "baz" +## 0 +function array_contains() { + local seeking=$1 + shift + local array=("$@") + shift + for element in "${array[@]}"; do + if [[ "${element}" == "${seeking}" ]]; then + return 0 + fi + done + return 1 +} + +# Version related functions + +VERSION_SED_REGEX="[0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\(-[a-zA-Z0-9]\{1,\}\)\{0,1\}" + +function validate_version() { + local input_version=$1 + shift + local parsed_version + parsed_version=$(sed -n "s/^v\{0,1\}\(${VERSION_SED_REGEX}\)$/\1/p" <<<"${input_version}") + printf %s "${parsed_version}" +} + +function retrieve_package_version() { + sed -n "s/^__version__[[:blank:]]*\=[[:blank:]]*\"\(${VERSION_SED_REGEX}\)\"/\1/p" "${ROOT_DIR}/cogment_lab/version.py" +} + +function update_package_version() { + local version=$1 + sed -i.bak "/^__version__[[:blank:]]*\=/s/${VERSION_SED_REGEX}/${version}/" "${ROOT_DIR}/cogment_lab/version.py" + retrieve_package_version +} diff --git a/scripts/create_release_branch.sh b/scripts/create_release_branch.sh new file mode 100755 index 0000000..eb25a87 --- /dev/null +++ b/scripts/create_release_branch.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +#### +# Release preparation script +# +# Should be executed by the release manager to initiate the finalization work on a particular release +#### + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/commons.sh" + +function usage() { + local usage_str="" + usage_str+="Update the version and create a release branch\n\n" + usage_str+="Usage:\n" + usage_str+=" $(basename "${BASH_SOURCE[0]}") [--dry-run]\n\n" + usage_str+=" version: looks like MAJOR.MINOR.PATCH[.PRERELEASE] having:\n" + usage_str+=" - MAJOR, MINOR and PATCH digit only,\n" + usage_str+=" - PRERELEASE optional, any string >1 alphanumerical characters.\n\n" + usage_str+="Options:\n" + usage_str+=" --dry-run: Do not push anything to the remote.\n" + usage_str+=" -h, --help: Show this screen.\n" + printf "%b" "${usage_str}" +} + +set -o errexit + +# Parse the command line arguments. +dry_run=0 + +while [[ "$1" != "" ]]; do + case $1 in + --dry-run) + dry_run=1 + ;; + --help | -h) + usage + exit 0 + ;; + *) + if [[ -z "${version}" ]]; then + input_version=$1 + validated_version=$(validate_version "${input_version}") + if [[ -z "${validated_version}" ]]; then + printf "%s: provided version is invalid.\n\n" "${input_version}" + usage + exit 1 + fi + version="${validated_version}" + else + printf "%s: unrecognized argument.\n\n" "$1" + usage + exit 1 + fi + ;; + esac + shift +done + +if [[ -z "${version}" ]]; then + printf "Missing version.\n\n" + usage + exit 1 +fi + +printf "* Preparing release v%s...\n" "${version}" + +# Move to the remote `develop` branch +git -C "${ROOT_DIR}" fetch -q "${GIT_REMOTE}" +git -C "${ROOT_DIR}" checkout -q "${GIT_REMOTE}/develop" + +printf "** Now on the latest %s/develop\n" "${GIT_REMOTE}" + +# Retrieving the current version of the package +current_version=$(retrieve_package_version) + +printf "** Current version is v%s\n" "${current_version}" + +# Creating the release branch, this will fail if the branch already exists +release_branch="release/v${version}" +git -C "${ROOT_DIR}" branch "${release_branch}" +git -C "${ROOT_DIR}" checkout -q "${release_branch}" + +printf "** Release branch \"%s\" created\n" "${release_branch}" + +# Updating the version of the package and commit +updated_version=$(update_package_version "${version}") +if [[ "${version}" != "${updated_version}" ]]; then + printf "Error while updating the version to %s.\n" "${version}" + exit 1 +fi +git -C "${ROOT_DIR}" commit -q -a -m"Preparing release v${version}" + +# TODO here we could ask / retrieve the latest version of the internal dependencies and update .cogment-api.yaml and .gitlab-ci.yml + +printf "** Version updated to v%s\n" "${updated_version}" + +if [[ "${dry_run}" == 1 ]]; then + printf "** DRY RUN SUCCESSFUL - Nothing pushed to %s\n" "${GIT_REMOTE}" +else + git -C "${ROOT_DIR}" push -q "${GIT_REMOTE}" "${release_branch}" + printf "** Release branch \"%s\" pushed to \"%s\" \n" "${release_branch}" "${GIT_REMOTE}" +fi + +printf "* To finalize the release:\n" +printf "** Update the dependencies, in particular make sure nothing is relying on a \"latest\" version of another package.\n" +printf "** Check and update the package's changelog at \"%s/CHANGELOG.md\"\n" "${ROOT_DIR}" +printf "** Make sure the CI builds everything properly\n" +printf "** Finally, run \"%s %s\"\n" "$(dirname "${BASH_SOURCE[0]}")/tag_release.sh" "${version}" diff --git a/scripts/tag_release.sh b/scripts/tag_release.sh new file mode 100755 index 0000000..d72dcf3 --- /dev/null +++ b/scripts/tag_release.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +#### +# Release finalization script +# +# Should be executed by the release manager to finalize a release and re-init the develop branch +#### + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/commons.sh" + +function usage() { + local usage_str="" + usage_str+="Tag a release: finalize the changelog, merge the release branch, tag the release and rebase the develop branch\n\n" + usage_str+="Usage:\n" + usage_str+=" $(basename "${BASH_SOURCE[0]}") [--dry-run]\n\n" + usage_str+=" version: looks like MAJOR.MINOR.PATCH[.PRERELEASE] having:\n" + usage_str+=" - MAJOR, MINOR and PATCH digit only,\n" + usage_str+=" - PRERELEASE optional, any string >1 alphanumerical characters.\n\n" + usage_str+="Options:\n" + usage_str+=" --dry-run: Do not push anything to git.\n" + usage_str+=" -h, --help: Show this screen.\n" + printf "%b" "${usage_str}" +} + +# Parse the commande line arguments. +dry_run=0 + +while [[ "$1" != "" ]]; do + case $1 in + --dry-run) + dry_run=1 + ;; + --help | -h) + usage + exit 0 + ;; + *) + if [[ -z "${version}" ]]; then + input_version=$1 + validated_version=$(validate_version "${input_version}") + if [[ -z "${validated_version}" ]]; then + printf "%s: provided version is invalid.\n\n" "${input_version}" + usage + exit 1 + fi + version="${validated_version}" + else + printf "%s: unrecognized argument.\n\n" "$1" + usage + exit 1 + fi + ;; + esac + shift +done + +if [[ -z "${version}" ]]; then + printf "Missing version.\n\n" + usage + exit 1 +fi + +printf "* Finalizing release v%s...\n" "${version}" + +# Move to the remote release branch +release_branch="release/v${version}" +git -C "${ROOT_DIR}" fetch -q "${GIT_REMOTE}" +git -C "${ROOT_DIR}" checkout -q -B "${release_branch}" "${GIT_REMOTE}/${release_branch}" + +printf "** Now on the latest commit for release branch \"%s/%s\"\n" "${GIT_REMOTE}" "${release_branch}" + +# Check the package version +package_version=$(retrieve_package_version) +if [[ "${version}" != "${package_version}" ]]; then + printf "Package version, %s, doesn't match.\n" "${version}" + exit 1 +fi + +printf "** Package version checked\n" + +# Update the changelog and commit +changelog_md_file="${ROOT_DIR}/CHANGELOG.md" +today=$(date +%Y-%m-%d) +sed -i.bak "s/.*##\ Unreleased.*/## Unreleased${SED_NL}${SED_NL}## v${version} - ${today}/g" "${changelog_md_file}" +git -C "${ROOT_DIR}" commit -q -a -m"Finalizing release v${version}" + +printf "** \"%s\" updated and committed\n" "${changelog_md_file}" + +# Move to the remote main branch +git -C "${ROOT_DIR}" checkout -q -B main "${GIT_REMOTE}"/main + +printf "** Now on the latest commit for main branch \"%s/main\"\n" "${GIT_REMOTE}" + +# Fast forward merge! +git -C "${ROOT_DIR}" merge -q --ff-only "${release_branch}" + +printf "** \"%s\" fast forward merged in \"main\"\n" "${GIT_REMOTE}" + +# Tag ! +git -C "${ROOT_DIR}" tag -a "v${version}" -m "v${version}" + +# Push main and the tag to the remote +if [[ "${dry_run}" == 1 ]]; then + printf "** DRY RUN SUCCESSFUL - Skipping pushing \"main\" branch and tags to %s \n" "${GIT_REMOTE}" +else + git -C "${ROOT_DIR}" push -q --tags "${GIT_REMOTE}" main + printf "** \"main\" branch pushed to \"%s\" \n" "${GIT_REMOTE}" +fi + +# Move to the remote develop branch +git -C "${ROOT_DIR}" checkout -q -B develop "${GIT_REMOTE}"/develop + +printf "** Now on the latest commit for develop branch \"%s/develop\"\n" "${GIT_REMOTE}" + +# Rebase develop on the latest merge +git -C "${ROOT_DIR}" rebase -q main + +printf "** \"develop\" rebased on the just release \"main\"\n" + +# TODO here we switch back to the "latest" version of the internal dependencies and update .cogment-api.yaml and .gitlab-ci.yml + +# (Force) push develop to the remote +if [[ "${dry_run}" == 1 ]]; then + printf "** DRY RUN SUCCESSFUL - Skipping (force) pushing \"develop\" branch to %s \n" "${GIT_REMOTE}" +else + git -C "${ROOT_DIR}" push -q -f "${GIT_REMOTE}" develop + printf "** \"develop\" branch (force) pushed to \"%s\" \n" "${GIT_REMOTE}" +fi diff --git a/setup.py b/setup.py index d50072a..af0119f 100644 --- a/setup.py +++ b/setup.py @@ -24,13 +24,13 @@ def get_version(): """Gets the cogment_lab version.""" - path = CWD / "cogment_lab" / "__init__.py" + path = CWD / "cogment_lab" / "version.py" content = path.read_text() for line in content.splitlines(): if line.startswith("__version__"): return line.strip().split()[-1].strip().strip('"') - raise RuntimeError("bad version data in __init__.py") + raise RuntimeError("bad version data in 'cogment_lab/version.py'") def get_description():