Skip to content

Add additional WriteMultipart tests (#5743) (#5746) #3307

Add additional WriteMultipart tests (#5743) (#5746)

Add additional WriteMultipart tests (#5743) (#5746) #3307

Workflow file for this run

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
# tests for `object_store` crate
name: object_store
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
# trigger for all PRs that touch certain files and changes to master
on:
push:
branches:
- master
pull_request:
paths:
- object_store/**
- .github/**
jobs:
clippy:
name: Clippy
runs-on: ubuntu-latest
container:
image: amd64/rust
defaults:
run:
working-directory: object_store
steps:
- uses: actions/checkout@v4
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
- name: Setup Clippy
run: rustup component add clippy
# Run different tests for the library on its own as well as
# all targets to ensure that it still works in the absence of
# features that might be enabled by dev-dependencies of other
# targets.
- name: Run clippy with default features
run: cargo clippy -- -D warnings
- name: Run clippy with aws feature
run: cargo clippy --features aws -- -D warnings
- name: Run clippy with gcp feature
run: cargo clippy --features gcp -- -D warnings
- name: Run clippy with azure feature
run: cargo clippy --features azure -- -D warnings
- name: Run clippy with http feature
run: cargo clippy --features http -- -D warnings
- name: Run clippy with all features
run: cargo clippy --all-features -- -D warnings
- name: Run clippy with all features and all targets
run: cargo clippy --all-features --all-targets -- -D warnings
# test doc links still work
#
# Note that since object_store is not part of the main workspace,
# this needs a separate docs job as it is not covered by
# `cargo doc --workspace`
docs:
name: Rustdocs
runs-on: ubuntu-latest
defaults:
run:
working-directory: object_store
env:
RUSTDOCFLAGS: "-Dwarnings"
steps:
- uses: actions/checkout@v4
- name: Run cargo doc
run: cargo doc --document-private-items --no-deps --workspace --all-features
# test the crate
# This runs outside a container to workaround lack of support for passing arguments
# to service containers - https://github.com/orgs/community/discussions/26688
linux-test:
name: Emulator Tests
runs-on: ubuntu-latest
defaults:
run:
working-directory: object_store
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
RUST_BACKTRACE: "1"
# Run integration tests
TEST_INTEGRATION: 1
EC2_METADATA_ENDPOINT: http://localhost:1338
AZURE_CONTAINER_NAME: test-bucket
AZURE_STORAGE_USE_EMULATOR: "1"
AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001"
AWS_BUCKET: test-bucket
AWS_DEFAULT_REGION: "us-east-1"
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ALLOW_HTTP: true
AWS_COPY_IF_NOT_EXISTS: dynamo:test-table:2000
AWS_CONDITIONAL_PUT: dynamo:test-table:2000
AWS_SERVER_SIDE_ENCRYPTION: aws:kms
HTTP_URL: "http://localhost:8080"
GOOGLE_BUCKET: test-bucket
GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
steps:
- uses: actions/checkout@v4
# We are forced to use docker commands instead of service containers as we need to override the entrypoints
# which is currently not supported - https://github.com/actions/runner/discussions/1872
- name: Configure Fake GCS Server (GCP emulation)
# Custom image - see fsouza/fake-gcs-server#1164
run: |
echo "GCS_CONTAINER=$(docker run -d -p 4443:4443 tustvold/fake-gcs-server -scheme http -backend memory -public-host localhost:4443)" >> $GITHUB_ENV
# Give the container a moment to start up prior to configuring it
sleep 1
curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": "", "private_key_id": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
- name: Setup WebDav
run: docker run -d -p 8080:80 rclone/rclone serve webdav /data --addr :80
- name: Setup LocalStack (AWS emulation)
run: |
echo "LOCALSTACK_CONTAINER=$(docker run -d -p 4566:4566 localstack/localstack:3.3.0)" >> $GITHUB_ENV
echo "EC2_METADATA_CONTAINER=$(docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2)" >> $GITHUB_ENV
aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
aws --endpoint-url=http://localhost:4566 dynamodb create-table --table-name test-table --key-schema AttributeName=path,KeyType=HASH AttributeName=etag,KeyType=RANGE --attribute-definitions AttributeName=path,AttributeType=S AttributeName=etag,AttributeType=S --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5
KMS_KEY=$(aws --endpoint-url=http://localhost:4566 kms create-key --description "test key")
echo "AWS_SSE_KMS_KEY_ID=$(echo $KMS_KEY | jq -r .KeyMetadata.KeyId)" >> $GITHUB_ENV
- name: Configure Azurite (Azure emulation)
# the magical connection string is from
# https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
run: |
echo "AZURITE_CONTAINER=$(docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite)" >> $GITHUB_ENV
az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;'
- name: Setup Rust toolchain
run: |
rustup toolchain install stable
rustup default stable
- name: Run object_store tests
run: cargo test --features=aws,azure,gcp,http
- name: GCS Output
if: ${{ !cancelled() }}
run: docker logs $GCS_CONTAINER
- name: LocalStack Output
if: ${{ !cancelled() }}
run: docker logs $LOCALSTACK_CONTAINER
- name: EC2 Metadata Output
if: ${{ !cancelled() }}
run: docker logs $EC2_METADATA_CONTAINER
- name: Azurite Output
if: ${{ !cancelled() }}
run: docker logs $AZURITE_CONTAINER
# test the object_store crate builds against wasm32 in stable rust
wasm32-build:
name: Build wasm32
runs-on: ubuntu-latest
container:
image: amd64/rust
defaults:
run:
working-directory: object_store
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
target: wasm32-unknown-unknown,wasm32-wasi
- name: Build wasm32-unknown-unknown
run: cargo build --target wasm32-unknown-unknown
- name: Build wasm32-wasi
run: cargo build --target wasm32-wasi