From fa1b20b12d6d3c973d4c1c68c4dd63cdeb3e99da Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 11 Apr 2024 19:49:32 +0200 Subject: [PATCH] test(gateway): with-remote-block-backend --- .github/workflows/gateway-conformance.yml | 141 +++++++++++++++++----- examples/gateway/car-file/README.md | 5 +- gateway/backend_car_fetcher.go | 3 +- gateway/blockstore.go | 3 +- 4 files changed, 118 insertions(+), 34 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index d5461e443..0928d2650 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -1,19 +1,24 @@ name: Gateway Conformance +# This workflow runs https://github.com/ipfs/gateway-conformance +# against different backend implementations of boxo/gateway on: push: branches: - main pull_request: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: true jobs: - gateway-conformance-car-file: + # This test uses a static CAR file as a local blockstore, + # allowing us to test conformance against BlocksBackend (gateway/backend_blocks.go) + # which is used by implementations like Kubo + local-block-backend: runs-on: ubuntu-latest - name: Gateway Conformance (CAR File Gateway) steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures @@ -23,21 +28,22 @@ jobs: merged: true # 2. Build the gateway binary - - name: Setup Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - name: Checkout boxo uses: actions/checkout@v4 with: path: boxo - - name: Build car-gateway - run: go build -o gateway + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'boxo/examples/go.mod' + cache-dependency-path: "boxo/**/*.sum" + - name: Build test-gateway + run: go build -o test-gateway working-directory: boxo/examples/gateway/car-file # 3. Start the gateway binary - - name: Start car-gateway - run: boxo/examples/gateway/car-file/gateway -c fixtures/fixtures.car -p 8040 & + - name: Start test-gateway + run: boxo/examples/gateway/car-file/test-gateway -c fixtures/fixtures.car -p 8040 & # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests @@ -57,20 +63,24 @@ jobs: run: cat output.md >> $GITHUB_STEP_SUMMARY - name: Upload HTML report if: failure() || success() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: gateway-conformance.html + name: gateway-conformance_local-block-backend.html path: output.html - name: Upload JSON report if: failure() || success() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: gateway-conformance.json + name: gateway-conformance_local-block-backend.json path: output.json - gateway-conformance-remote-car: + # This test uses remote block gateway (?format=raw) as a remote blockstore, + # allowing us to test conformance against RemoteBlocksBackend + # (gateway/backend_blocks.go) which is used by implementations like + # rainbow configured to use with remote block backend + # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw + remote-block-backend: runs-on: ubuntu-latest - name: Gateway Conformance (Remote CAR Gateway) steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures @@ -80,32 +90,101 @@ jobs: merged: true # 2. Build the gateway binaries + - name: Checkout boxo + uses: actions/checkout@v4 + with: + path: boxo - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 + with: + go-version-file: 'boxo/examples/go.mod' + cache-dependency-path: "boxo/**/*.sum" + - name: Build remote-block-backend # it will act as a trustless CAR gateway + run: go build -o remote-block-backend + working-directory: boxo/examples/gateway/car-file + - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote block backend + run: go build -o test-gateway + working-directory: boxo/examples/gateway/proxy-blocks + + # 3. Start the gateway binaries + - name: Start remote HTTP backend that serves application/vnd.ipld.raw + run: boxo/examples/gateway/car-file/remote-block-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.car requests + - name: Start gateway that uses the remote block backend + run: boxo/examples/gateway/proxy-blocks/test-gateway -g http://127.0.0.1:8030 -p 8040 & + + # 4. Run the gateway-conformance tests + - name: Run gateway-conformance tests + uses: ipfs/gateway-conformance/.github/actions/test@v0.5 + with: + gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway + json: output.json + xml: output.xml + html: output.html + markdown: output.md + subdomain-url: http://example.net + specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway + args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore' + + # 5. Upload the results + - name: Upload MD summary + if: failure() || success() + run: cat output.md >> $GITHUB_STEP_SUMMARY + - name: Upload HTML report + if: failure() || success() + uses: actions/upload-artifact@v4 with: - go-version: 1.21.x + name: gateway-conformance_remote-block-backend.html + path: output.html + - name: Upload JSON report + if: failure() || success() + uses: actions/upload-artifact@v4 + with: + name: gateway-conformance_remote-block-backend.json + path: output.json + + # This test uses remote CAR gateway (?format=car, IPIP-402) + # allowing us to test conformance against remote CarFetcher backend. + # (gateway/backend_car_fetcher.go) which is used by implementations like + # rainbow configured to use with remote car backend + # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#car-responses-application-vnd-ipld-car + remote-car-backend: + runs-on: ubuntu-latest + steps: + # 1. Download the gateway-conformance fixtures + - name: Download gateway-conformance fixtures + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.5 + with: + output: fixtures + merged: true + + # 2. Build the gateway binaries - name: Checkout boxo uses: actions/checkout@v4 with: path: boxo - - name: Build car-gateway - run: go build -o gateway + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'boxo/examples/go.mod' + cache-dependency-path: "boxo/**/*.sum" + - name: Build remote-car-backend # it will act as a trustless CAR gateway + run: go build -o remote-car-backend working-directory: boxo/examples/gateway/car-file - - name: Build proxy-car-gateway - run: go build -o gateway + - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote CAR backend + run: go build -o test-gateway working-directory: boxo/examples/gateway/proxy-car # 3. Start the gateway binaries - - name: Start car-gateway - run: boxo/examples/gateway/car-file/gateway -c fixtures/fixtures.car -p 8030 & - - name: Start proxy-car-gateway - run: boxo/examples/gateway/proxy-car/gateway -g http://127.0.0.1:8030 -p 8040 & + - name: Start remote HTTP backend that serves application/vnd.ipld.car (IPIP-402) + run: boxo/examples/gateway/car-file/remote-car-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.raw requests + - name: Start gateway that uses the remote CAR backend + run: boxo/examples/gateway/proxy-car/test-gateway -g http://127.0.0.1:8030 -p 8040 & # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests uses: ipfs/gateway-conformance/.github/actions/test@v0.5 with: - gateway-url: http://127.0.0.1:8040 + gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway json: output.json xml: output.xml html: output.html @@ -120,13 +199,13 @@ jobs: run: cat output.md >> $GITHUB_STEP_SUMMARY - name: Upload HTML report if: failure() || success() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: gateway-conformance.html + name: gateway-conformance_remote-car-backend.html path: output.html - name: Upload JSON report if: failure() || success() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: gateway-conformance.json + name: gateway-conformance_remote-car-backend.json path: output.json diff --git a/examples/gateway/car-file/README.md b/examples/gateway/car-file/README.md index 5a95063d4..2645d7b17 100644 --- a/examples/gateway/car-file/README.md +++ b/examples/gateway/car-file/README.md @@ -1,9 +1,12 @@ -# HTTP Gateway backed by a CAR File +# HTTP Gateway backed by a CAR File as BlocksBackend This is an example that shows how to build a Gateway backed by the contents of a CAR file. A [CAR file](https://ipld.io/specs/transport/car/) is a Content Addressable aRchive that contains blocks. +The `main.go` sets up a `blockService` backed by a static CAR file, +and then uses it to initialize `gateway.NewBlocksBackend(blockService)`. + ## Build ```bash diff --git a/gateway/backend_car_fetcher.go b/gateway/backend_car_fetcher.go index ec348694a..cf9d2ec04 100644 --- a/gateway/backend_car_fetcher.go +++ b/gateway/backend_car_fetcher.go @@ -29,9 +29,10 @@ type remoteCarFetcher struct { } // NewRemoteCarFetcher returns a [CarFetcher] that is backed by one or more gateways -// that support partial CAR requests, as described in [IPIP-402]. You can optionally +// that support partial [CAR requests], as described in [IPIP-402]. You can optionally // pass your own [http.Client]. // +// [CAR requests]: https://www.iana.org/assignments/media-types/application/vnd.ipld.car // [IPIP-402]: https://specs.ipfs.tech/ipips/ipip-0402 func NewRemoteCarFetcher(gatewayURL []string, httpClient *http.Client) (CarFetcher, error) { if len(gatewayURL) == 0 { diff --git a/gateway/blockstore.go b/gateway/blockstore.go index c1c4b067b..89ae0a27d 100644 --- a/gateway/blockstore.go +++ b/gateway/blockstore.go @@ -158,10 +158,11 @@ type remoteBlockstore struct { } // NewRemoteBlockstore creates a new [blockstore.Blockstore] that is backed by one -// or more gateways that support RAW block requests. See the [Trustless Gateway] +// or more gateways that support [RAW block] requests. See the [Trustless Gateway] // specification for more details. You can optionally pass your own [http.Client]. // // [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/ +// [RAW block]: https://www.iana.org/assignments/media-types/application/vnd.ipld.raw func NewRemoteBlockstore(gatewayURL []string, httpClient *http.Client) (blockstore.Blockstore, error) { if len(gatewayURL) == 0 { return nil, errors.New("missing gateway URLs to which to proxy")