From ee8b5f88079436cbdabe21fff94b9daf1c20187f Mon Sep 17 00:00:00 2001 From: Oleg Kulachenko Date: Tue, 3 Oct 2023 20:11:58 +0400 Subject: [PATCH 1/8] .github: Add CODEOWNERS Signed-off-by: Oleg Kulachenko --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..2112c74 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @vvarg229 From f1b85476bbf836cda82a9cd1f5ce504bfaa69d46 Mon Sep 17 00:00:00 2001 From: Oleg Kulachenko Date: Fri, 29 Sep 2023 01:01:18 +0400 Subject: [PATCH 2/8] Add .gitignore Signed-off-by: Oleg Kulachenko --- .gitignore | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6220a21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# ignore IDE files +.vscode +.idea + +.DS_Store + + +# ignore test results +**/log.html +**/output.xml +**/report.html +**/dockerlogs*.tar.gz +allure_results/* +xunit_results.xml + +# ignore caches under any path +**/__pycache__ +**/.pytest_cache + +# ignore work directories and setup files +.setup +.env +TemporaryDir/* +artifacts/* +venv.*/* +/tests/venv +/*wallet_config.yml From b49ab15009237a620fc8a14dc76d2129f8a57c29 Mon Sep 17 00:00:00 2001 From: Oleg Kulachenko Date: Fri, 29 Sep 2023 00:46:11 +0400 Subject: [PATCH 3/8] Add script to push files to NeoFS Signed-off-by: Oleg Kulachenko --- push-to-neofs.py | 189 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 190 insertions(+) create mode 100644 push-to-neofs.py create mode 100644 requirements.txt diff --git a/push-to-neofs.py b/push-to-neofs.py new file mode 100644 index 0000000..350f34c --- /dev/null +++ b/push-to-neofs.py @@ -0,0 +1,189 @@ +import os +import subprocess +import argparse +import magic + +FILE_PATH = "FilePath" # the key for the attribute, is the path for the static page and allure report zip files +CONTENT_TYPE = "ContentType" +NEOFS_WALLET_PASSWORD_ENV_NAME = "NEOFS_WALLET_PASSWORD" +PORT_8080 = 8080 + + +def parse_args(): + parser = argparse.ArgumentParser(description="Process allure reports") + parser.add_argument( + "--neofs_domain", + required=True, + type=str, + help="NeoFS network domain, example: st1.storage.fs.neo.org", + ) + parser.add_argument("--wallet", required=True, type=str, help="Path to the wallet") + parser.add_argument("--cid", required=True, type=str, help="Container ID") + parser.add_argument( + "--attributes", + required=False, + type=str, + help="User attributes in form of Key1=Value1,Key2=Value2" + "For example, it's convenient to create url links to access an object via http:" + "FilePath=96-1697035975/dir/3.txt" + "Type=test_result,Master=true,RUN_ID=96-1697035975", + default=None, + ) + parser.add_argument( + "--url_path_prefix", + required=False, + type=str, + help="This is a prefix to the url address for each of the files(objects)." + "For example, if Container ID is HXSaMJXk2g8C14ht8HSi7BBaiYZ1HeWh2xnWPGQCg4H6 and" + "--url_path_prefix is '96-1697035975', then the url will be:" + " https://http.fs.neo.org/HXSaMJXk2g8C14ht8HSi7BBaiYZ1HeWh2xnWPGQCg4H6/832-1695916423/file.txt" + "Without --url_path_prefix the url will be:" + " https://http.fs.neo.org/HXSaMJXk2g8C14ht8HSi7BBaiYZ1HeWh2xnWPGQCg4H6/file.txt", + default=None, + ) + parser.add_argument( + "--files-dir", + required=True, + type=str, + help="Path to the directory with the files to be pushed", + ) + parser.add_argument( + "--lifetime", + required=False, + type=int, + help="Lifetime in epochs - number of epochs for object to stay valid. If provided, will be added to the " + "current epoch to calculate expiration epoch. If not provided, or if it is 0, the report will be stored " + "indefinitely", + default=None, + ) + parser.add_argument( + "--put-timeout", + required=False, + type=int, + help="Timeout for the put each file to neofs, in seconds. Default is 600 seconds", + default=600, + ) + + return parser.parse_args() + + +def get_password() -> str: + password = os.getenv(NEOFS_WALLET_PASSWORD_ENV_NAME) + return password + + +def get_current_epoch(endpoint: str) -> int: + cmd = f"neofs-cli netmap epoch --rpc-endpoint {endpoint}" + epoch_str = subprocess.check_output(cmd, shell=True).strip().decode("utf-8") + return int(epoch_str) + + +def get_rpc_endpoint(neofs_domain: str) -> str: + return f"{neofs_domain}:{PORT_8080}" + + +def push_file( + directory: str, + subdir: str, + url_path_prefix: str, + filename: str, + attributes: str, + base_cmd: str, + put_timeout: int, +) -> None: + filepath = os.path.join(subdir, filename) + mime_type = magic.from_file(filepath, mime=True) + relative_path = os.path.relpath(filepath, os.path.dirname(directory)) + + if url_path_prefix is not None: + neofs_path_attr = os.path.join(url_path_prefix, relative_path) + else: + neofs_path_attr = relative_path + + base_cmd_with_file = f"{base_cmd} --file {filepath} --attributes {FILE_PATH}={neofs_path_attr},{CONTENT_TYPE}={mime_type}" + + if attributes is not None: + base_cmd_with_file += f",{attributes}" + + print(f"Neofs cli cmd is: {base_cmd_with_file}") + + try: + compl_proc = subprocess.run( + base_cmd_with_file, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=put_timeout, + shell=True, + ) + + print(f"RC: {compl_proc.returncode}") + print(f"Output: {compl_proc.stdout}") + print(f"Error: {compl_proc.stderr}") + + except subprocess.CalledProcessError as e: + raise Exception( + f"Command failed: {e.cmd}\n" + f"Error code: {e.returncode}\n" + f"Output: {e.output}\n" + f"Stdout: {e.stdout}\n" + f"Stderr: {e.stderr}\n" + ) + + +def push_files_to_neofs( + directory: str, + endpoint: str, + wallet: str, + cid: str, + attributes: str, + url_path_prefix: str, + lifetime: int, + put_timeout: int, + password: str, +) -> None: + if not os.path.exists(directory): + raise Exception(f"Directory '{directory}' does not exist.") + if not os.listdir(directory): + raise Exception(f"Directory '{directory}' is empty.") + + base_cmd = ( + f"NEOFS_CLI_PASSWORD={password} neofs-cli --rpc-endpoint {endpoint} " + f"--wallet {wallet} object put --cid {cid} --timeout {put_timeout}s" + ) + if lifetime is not None and lifetime > 0: + current_epoch = get_current_epoch(endpoint) + expiration_epoch = current_epoch + lifetime + base_cmd += f" --expire-at {expiration_epoch}" + + base_path = os.path.abspath(directory) + for subdir, dirs, files in os.walk(base_path): + for filename in files: + push_file( + base_path, + subdir, + url_path_prefix, + filename, + attributes, + base_cmd, + put_timeout, + ) + + +if __name__ == "__main__": + args = parse_args() + neofs_password = get_password() + rpc_endpoint = get_rpc_endpoint(args.neofs_domain) + + push_files_to_neofs( + args.files_dir, + rpc_endpoint, + args.wallet, + args.cid, + args.attributes, + args.url_path_prefix, + args.lifetime, + args.put_timeout, + neofs_password, + ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..952a07d --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +python-magic==0.4.27 From cfec967d8d741ba4461f7dbd6f2eb18e101223cb Mon Sep 17 00:00:00 2001 From: Oleg Kulachenko Date: Fri, 29 Sep 2023 00:59:58 +0400 Subject: [PATCH 4/8] Add action to push files to NeoFS Signed-off-by: Oleg Kulachenko --- action.yml | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 action.yml diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..91a7e53 --- /dev/null +++ b/action.yml @@ -0,0 +1,104 @@ +name: 'Publish to NeoFS' +description: 'This action allows you to publish files, tests reports, static web pages and etc. to NeoFS' +author: 'Neo SPCC' +branding: + icon: 'server' + color: 'green' +inputs: + NEOFS_WALLET: + description: Base64-encoded NEP-6 Neo N3 wallet for NeoFS network access. See the README.md for more information + required: true + NEOFS_WALLET_PASSWORD: + description: N3 wallet password + required: true + NEOFS_NETWORK_DOMAIN: + description: Rpc endpoint domain address + required: false + default: 'st1.storage.fs.neo.org' + NEOFS_HTTP_GATE: + description: HTTP Gateway domain address + required: false + default: 'http.fs.neo.org' + STORE_OBJECTS_CID: + description: Container ID for your data + required: true + PATH_TO_FILES_DIR: + description: Path to the directory with the files to be pushed + required: true + NEOFS_ATTRIBUTES: + description: User attributes in form of Key1=Value1,Key2=Value2 + required: false + URL_PREFIX: + description: Prefix to the url address for each of the files(objects) + required: false + LIFETIME: + description: Number of epochs for object to stay valid + required: false + default: 0 + +outputs: + OUTPUT_CONTAINER_URL: + description: Container URL + value: ${{ steps.put_files.outputs.container_url }} + +runs: + using: composite + steps: + - name: Download latest stable neofs-cli for uploading reports to NeoFS + uses: dsaltares/fetch-gh-release-asset@1.1.1 + with: + repo: 'nspcc-dev/neofs-node' + version: 'tags/v0.38.1' + file: 'neofs-cli-amd64' + target: 'neofs/neofs-cli' + + - name: Chmod latest stable neofs-cli + shell: bash + run: | + sudo chmod a+x neofs-cli + working-directory: neofs + + - name: Enable stable neofs-cli + shell: bash + run: | + echo "$(pwd)" >> $GITHUB_PATH + working-directory: neofs + + - name: Create wallet + shell: bash + env: + NEOFS_WALLET: ${{ inputs.NEOFS_WALLET }} + run: | + echo "$NEOFS_WALLET" | base64 -d > wallet.json + + - name: Prepare venv + shell: bash + id: prepare_venv + run: | + python3 -m venv venv + source venv/bin/activate && pip install -r requirements.txt + + - name: Put files to NeoFS + shell: bash + id: put_files + env: + NEOFS_WALLET_PASSWORD: ${{ inputs.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ inputs.NEOFS_NETWORK_DOMAIN }} + STORE_OBJECTS_CID: ${{ inputs.STORE_OBJECTS_CID }} + NEOFS_HTTP_GATE: ${{ inputs.NEOFS_HTTP_GATE }} + PATH_TO_FILES_DIR: ${{ inputs.PATH_TO_FILES_DIR }} + NEOFS_ATTRIBUTES: ${{ inputs.NEOFS_ATTRIBUTES }} + URL_PREFIX: ${{ inputs.URL_PREFIX }} + LIFETIME: ${{ inputs.LIFETIME }} + run: | + source venv/bin/activate && NEOFS_CLI_PASSWORD=$NEOFS_WALLET_PASSWORD python ./push-to-neofs.py \ + --lifetime "$LIFETIME" --neofs_domain "$NEOFS_NETWORK_DOMAIN" --attributes "$NEOFS_ATTRIBUTES" \ + --cid "$STORE_OBJECTS_CID" --files-dir "$PATH_TO_FILES_DIR" --url_path_prefix "$URL_PREFIX" --wallet wallet.json + BASE_URL="https://$NEOFS_HTTP_GATE/$STORE_OBJECTS_CID" + if [ -z "$URL_PREFIX" ]; then + echo "container_url=$BASE_URL/" >> $GITHUB_OUTPUT + echo "$BASE_URL/" + else + echo "container_url=$BASE_URL/$URL_PREFIX/" >> $GITHUB_OUTPUT + echo "$BASE_URL/$URL_PREFIX/" + fi From 5e53b83ff2d7ba881f874ad7150815724c5f0faa Mon Sep 17 00:00:00 2001 From: Oleg Kulachenko Date: Tue, 3 Oct 2023 20:12:20 +0400 Subject: [PATCH 5/8] workflows: Add DCO check Signed-off-by: Oleg Kulachenko --- .github/workflows/dco.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/dco.yml diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml new file mode 100644 index 0000000..d462989 --- /dev/null +++ b/.github/workflows/dco.yml @@ -0,0 +1,11 @@ +name: DCO check + +on: + pull_request: + branches: + - master + - develop + +jobs: + dco: + uses: nspcc-dev/.github/.github/workflows/dco.yml@master From 0f2bbadc165ca5d785e0f3cd196a5a1551e891e5 Mon Sep 17 00:00:00 2001 From: Oleg Kulachenko Date: Tue, 3 Oct 2023 20:13:31 +0400 Subject: [PATCH 6/8] Add automated tests Signed-off-by: Oleg Kulachenko --- .github/workflows/tests.yml | 126 +++++++++++++++++++++++++++ tests/conftest.py | 23 +++++ tests/data/1.txt | 1 + tests/data/2.txt | 1 + tests/data/dir/3.txt | 1 + tests/data/dir/subdir/4.txt | 1 + tests/data/dir/subdir/subdir_2/5.txt | 1 + tests/requirements.txt | 3 + tests/test_downloads.py | 43 +++++++++ 9 files changed, 200 insertions(+) create mode 100644 .github/workflows/tests.yml create mode 100644 tests/conftest.py create mode 100644 tests/data/1.txt create mode 100644 tests/data/2.txt create mode 100644 tests/data/dir/3.txt create mode 100644 tests/data/dir/subdir/4.txt create mode 100644 tests/data/dir/subdir/subdir_2/5.txt create mode 100644 tests/requirements.txt create mode 100644 tests/test_downloads.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..916bcff --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,126 @@ +name: Run automated tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + - support/** + types: [opened, synchronize] + paths-ignore: + - '**/*.md' + release: + types: + - published + workflow_dispatch: + +env: + TESTS_DATA_DIR: './tests/data' + +permissions: write-all + +jobs: + run_system_tests: + runs-on: ubuntu-latest + timeout-minutes: 500 + steps: + - name: Get the current date + id: date + shell: bash + run: | + echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT + + - name: Set url prefix + shell: bash + env: + TIMESTAMP: ${{ steps.date.outputs.timestamp }} + run: | + echo "URL_PREFIX=${{ github.run_number }}-$TIMESTAMP" >> $GITHUB_ENV + + - name: Set attributes + shell: bash + env: + TIMESTAMP: ${{ steps.date.outputs.timestamp }} + run: | + echo "NEOFS_ATTRIBUTES=RunNumber=${{ github.run_number }}-$TIMESTAMP,Type=action_test_files" >> $GITHUB_ENV + + - name: Checkout gh-push-to-neofs + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11.6' + - run: python --version + + - name: Prepare venv + shell: bash + id: prepare_venv + run: | + python3 -m venv venv + source venv/bin/activate && pip install -r requirements.txt + working-directory: ./tests + + - name: Run gh-push-to-neofs with url prefix + id: gh_push_to_neofs_with_url_prefix + uses: ./ + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }} + NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }} + URL_PREFIX: ${{ env.URL_PREFIX }} + + - name: Run tests + env: + OUTPUT_CONTAINER_URL: ${{ steps.gh_push_to_neofs_with_url_prefix.outputs.OUTPUT_CONTAINER_URL }} + run: | + source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" + working-directory: ./tests + + - name: Prepare directory name for tests without url prefix + shell: bash + env: + TIMESTAMP: ${{ steps.date.outputs.timestamp }} + run: | + echo "PREFIX_DIR=${{ github.run_number }}-$TIMESTAMP-$(uuidgen)" >> $GITHUB_ENV + + - name: Create a directory tree for tests with url prefix + shell: bash + run: | + mkdir "$PREFIX_DIR" + + - name: Move files to the directory tree for tests with url prefix + shell: bash + env: + SOURCE_DIR: ${{ env.TESTS_DATA_DIR }} + DEST_DIR: ${{ env.PREFIX_DIR }} + run: | + rsync -av "$SOURCE_DIR" "$DEST_DIR" + + - name: Run gh-push-to-neofs without url prefix + id: gh_push_to_neofs_without_url_prefix + uses: ./ + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.PREFIX_DIR }} + NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }} + + - name: Run tests + env: + OUTPUT_CONTAINER_URL: ${{ steps.gh_push_to_neofs_without_url_prefix.outputs.OUTPUT_CONTAINER_URL }} + REPORT_DIR: ${{ env.PREFIX_DIR }} + run: | + source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" + working-directory: ./tests diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c1c8209 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--base_url", action="store", default=None, help="Base URL to test against" + ) + parser.addoption( + "--report_dir", + action="store", + default=None, + help="Directory combine report and zip files", + ) + + +@pytest.fixture +def base_url(request): + return request.config.getoption("--base_url") + + +@pytest.fixture +def report_dir(request): + return request.config.getoption("--report_dir") diff --git a/tests/data/1.txt b/tests/data/1.txt new file mode 100644 index 0000000..43dd47e --- /dev/null +++ b/tests/data/1.txt @@ -0,0 +1 @@ +one \ No newline at end of file diff --git a/tests/data/2.txt b/tests/data/2.txt new file mode 100644 index 0000000..f719efd --- /dev/null +++ b/tests/data/2.txt @@ -0,0 +1 @@ +two diff --git a/tests/data/dir/3.txt b/tests/data/dir/3.txt new file mode 100644 index 0000000..1d19714 --- /dev/null +++ b/tests/data/dir/3.txt @@ -0,0 +1 @@ +three \ No newline at end of file diff --git a/tests/data/dir/subdir/4.txt b/tests/data/dir/subdir/4.txt new file mode 100644 index 0000000..ea1f343 --- /dev/null +++ b/tests/data/dir/subdir/4.txt @@ -0,0 +1 @@ +four \ No newline at end of file diff --git a/tests/data/dir/subdir/subdir_2/5.txt b/tests/data/dir/subdir/subdir_2/5.txt new file mode 100644 index 0000000..011203e --- /dev/null +++ b/tests/data/dir/subdir/subdir_2/5.txt @@ -0,0 +1 @@ +five \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..8f33a46 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +requests==2.31.0 +pytest==7.4.2 + diff --git a/tests/test_downloads.py b/tests/test_downloads.py new file mode 100644 index 0000000..4175147 --- /dev/null +++ b/tests/test_downloads.py @@ -0,0 +1,43 @@ +import os +import requests +import pytest +from urllib.parse import urljoin + + +def download_file(url: str) -> str: + response = requests.get(url) + response.raise_for_status() + return response.text + + +@pytest.mark.parametrize( + "path", + [ + "data/1.txt", + "data/2.txt", + "data/dir/3.txt", + "data/dir/subdir/4.txt", + "data/dir/subdir/subdir_2/5.txt", + ], +) +def test_file_content(base_url, report_dir, path): + if base_url is None: + pytest.skip("base_url is not provided. Provide it using --base_url option.") + + if not base_url.endswith("/"): + base_url += "/" + full_url = base_url + if report_dir is not None: + full_url = urljoin(base_url, report_dir) + if not full_url.endswith("/"): + full_url += "/" + full_url = urljoin(full_url, path) + print(f"full_url: {full_url}") + + remote_content = download_file(full_url) + with open(path, "r") as local_file: + local_content = local_file.read() + + assert ( + remote_content == local_content + ), f"Contents of {full_url} and {path} do not match." From ed4b1acc9b65c0f2a38fe8f74ab221618971cd66 Mon Sep 17 00:00:00 2001 From: Oleg Kulachenko Date: Mon, 16 Oct 2023 18:31:35 +0400 Subject: [PATCH 7/8] .github: Add logos Signed-off-by: Oleg Kulachenko --- .github/logo_dark.svg | 129 +++++++++++++++++++++++++++++++++++++++++ .github/logo_light.svg | 129 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 .github/logo_dark.svg create mode 100644 .github/logo_light.svg diff --git a/.github/logo_dark.svg b/.github/logo_dark.svg new file mode 100644 index 0000000..f045d97 --- /dev/null +++ b/.github/logo_dark.svg @@ -0,0 +1,129 @@ + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.github/logo_light.svg b/.github/logo_light.svg new file mode 100644 index 0000000..b4da076 --- /dev/null +++ b/.github/logo_light.svg @@ -0,0 +1,129 @@ + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 65e49a9bfca637026be7b343849a524c207391ed Mon Sep 17 00:00:00 2001 From: Oleg Kulachenko Date: Wed, 27 Sep 2023 13:10:43 +0000 Subject: [PATCH 8/8] Add README.md Signed-off-by: Oleg Kulachenko --- README.md | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..c1cc554 --- /dev/null +++ b/README.md @@ -0,0 +1,164 @@ +

+ + + + NeoFS logo + +

+

+ NeoFS is a decentralized distributed object storage integrated with the NEO Blockchain. +

+ +# GitHub Action to Publish to NeoFS +This GitHub action allows you to save files as objects in the [NeoFS](https://fs.neo.org/). + +This way you can both publicly and privately save logs and test results, host web pages, and publish releases. + +[Here](https://neospcc.medium.com/neofs-t5-testnet-has-been-started-ae75c30e856b) is a good article on how to get +started using the NeoFS testnet, this may be useful if you have no experience with NeoFS and want to get started with +the test network. + +## Supported platforms +This action supports the following platforms: +- Linux x64 + +This action tested on the following runners: +- [Ubuntu 22.04 GitHub-hosted runners](https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md) + +# Configuration + +## GitHub secrets +The following Sensitive information must be passed as +[GitHub Actions secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions). +It is very important to use SECRETS and NOT variables, otherwise your wallet, password and token will be available to +the whole internet. + +| Key | Value | Required | Default | +|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------| +| `NEOFS_WALLET` | Base64-encoded NEP-6 Neo N3 wallet. To create N3 wallet: `neo-go wallet init -w wallet.json -a` The output of this command should be here: 'cat wallet.json | base64' | **Yes** | N/A | +| `NEOFS_WALLET_PASSWORD` | N3 wallet password | **Yes** | N/A | + +Please keep sensitive data safe. + +## GitHub environment variables + +### NeoFS network environment variables +The following variables must be passed as +[GitHub Actions vars context](https://docs.github.com/en/actions/learn-github-actions/variables#using-the-vars-context-to-access-configuration-variable-values) +or [GitHub Actions environment variables](https://docs.github.com/en/actions/learn-github-actions/variables). + +Up-to-date information about NeoFS network can be seen on https://status.fs.neo.org. + +If you are using the NeoFS mainnet, we recommend that you do not change `NEOFS_NETWORK_DOMAIN` +and `NEOFS_HTTP_GATE` environment variables. + +| Key | Value | Required | Default | +|------------------------|---------------------------------------------------------------------------------------|----------|------------------------| +| `NEOFS_NETWORK_DOMAIN` | Rpc endpoint domain address | **No** | st1.storage.fs.neo.org | +| `NEOFS_HTTP_GATE` | HTTP Gateway domain address | **No** | http.fs.neo.org | +| `STORE_OBJECTS_CID` | Container ID for your data. For example: 7gHG4HB3BrpFcH9BN3KMZg6hEETx4mFP71nEoNXHFqrv | **Yes** | N/A | + + +### Workflow environment variables +The following variables must be passed as +[GitHub Actions vars context](https://docs.github.com/en/actions/learn-github-actions/variables#using-the-vars-context-to-access-configuration-variable-values) +or [GitHub Actions environment variables](https://docs.github.com/en/actions/learn-github-actions/variables). + +| Key | Value | Required | Default | +|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------| +| `PATH_TO_FILES_DIR` | Path to the directory with the files to be pushed | **Yes** | N/A | +| `NEOFS_ATTRIBUTES` | User attributes in form of Key1=Value1,Key2=Value2. By default, each object contains attributes of relative path to the file and MIME type of the file. | **No** | N/A | +| `URL_PREFIX` | Prefix to the url address for each of the files(objects) | **No** | N/A | + +### Expiration period environment variables +The following variables must be passed as +[GitHub Actions vars context](https://docs.github.com/en/actions/learn-github-actions/variables#using-the-vars-context-to-access-configuration-variable-values) +or [GitHub Actions environment variables](https://docs.github.com/en/actions/learn-github-actions/variables). + +These environment variables are responsible for the storage time of the results in the storage network in epochs +(in the mainnet, an epoch is approximately equal to one hour, so we can assume that values are specified in HOURS). + +After the period is over, the data will be deleted. They are convenient to use for log rotation or test reports. + +They default to 0, in which case the data will be stored until they are manually deleted. +We recommend setting a reasonable and convenient for work expiration period, for example, a month (744 hours). + + +| Key | Value | Required | Default | +|------------|-------------------------------------------|----------|---------| +| `LIFETIME` | Number of epochs for object to stay valid | **No** | 0 | + +## Output + +| Key | Value | Required | Default | +|------------------------|-------------------------------------------------------------------------------------------------------------|----------|---------| +| `OUTPUT_CONTAINER_URL` | Output example: https://http.storage.fs.neo.org/HXSaMJXk2g8C14ht8HSi7BBaiYZ1HeWh2xnWPGQCg4H6/872-1696332227 | **No** | N/A | + +# Dependencies + +## Python +The GitHub runner must have Python 3 installed on it. + +You can install Python like this: +```yml +- name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11.6' +``` + +# Examples +## How to store files from the directory to NeoFS + +```yml +name: Publish to NeoFS +on: + push: + branches: [ master ] +jobs: + push-to-neofs: + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11.6' + + - uses: actions/checkout@v4 + - name: Publish to NeoFS + uses: nspcc-dev/gh-push-to-neofs@master + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.PATH_TO_FILES_DIR }} +``` + +## How to store Allure report to NeoFS as static page + +See https://github.com/nspcc-dev/gh-push-allure-report-to-neofs for more details. + +In the [NeoFS](https://github.com/nspcc-dev/neofs-node) project, we use the following workflow to store the +[Allure report](https://github.com/allure-framework/allure2) as a static page in the NeoFS mainnet. +This is a good example of practical use of this action. + +To avoid creating a huge (weighing hundreds of megabytes or more) web page, in this example we upload zip archives with +attachments as separate objects. +Access to them from the Allure report will be via hyperlinks from the report page. Yes, this is the Web 1.0 world +in the Web 3.0. And it's beautiful. + +We use the [simple-elf/allure-report-action](https://github.com/simple-elf/allure-report-action) action to generate +the Allure report and the [allure-combine](https://github.com/MihanEntalpo/allure-single-html-file) tool to convert +the report to a static page. +Of course, you can use any other tool to generate the Allure report and convert it to a static page. For example, you +can use [allure-commandline](https://github.com/allure-framework/allure-npm) or Allure itself according to +[this](https://github.com/allure-framework/allure2/pull/2072) merged pull request. + +The Allure report will be available in a web browser at a link like this: +https://http.fs.neo.org/86C4P6uJC7gb5n3KkwEGpXRfdczubXyRNW5N9KeJRW73/53-1696453127/comb_report/complete.html# + +Attachments will also be available at the link: +https://http.fs.neo.org/86C4P6uJC7gb5n3KkwEGpXRfdczubXyRNW5N9KeJRW73/876-1696502182/comb_report/data/attachments/ce0fa9e280851f32.zip