Merge branch 'main' into vic/feat/legacy-claim-cache

storacha · Dec 19, 2024 · 7f202e9 · 7f202e9
2 parents 5e4ba01 + 7eb9500
commit 7f202e9
Show file tree

Hide file tree

Showing 21 changed files with 314 additions and 71 deletions.
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
@@ -1,7 +1,7 @@
 name: GoReleaser
 on:
   release:
-    types: [ published ]
+    types: [published]
   workflow_dispatch:
   workflow_run:
     workflows: [Releaser]
@@ -18,9 +18,9 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.20.x"
+          go-version: "1.23.x"
       - name: Release Binaries
-        uses: goreleaser/goreleaser-action@v5
+        uses: goreleaser/goreleaser-action@v6
         with:
           distribution: goreleaser
           version: latest

diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml
@@ -13,6 +13,10 @@ on:
       - "pkg/**"
   pull_request:
     branches: ["main"]
+  workflow_run:
+    workflows: [Releaser]
+    types: [completed]
+    branches: ["main"]
   workflow_dispatch:
 
 concurrency:
@@ -42,27 +46,19 @@ jobs:
       - uses: opentofu/setup-opentofu@v1
       - uses: actions/setup-go@v5
 
+      # always deploy to staging
       - name: Set Staging Environment Variables
-        if: startsWith(github.ref, 'refs/tags/') != true
         run: |
           echo "ENV=staging" >> $GITHUB_ENV
           echo "TF_WORKSPACE=staging" >> $GITHUB_ENV
           echo "TF_VAR_private_key=${{ secrets.STAGING_PRIVATE_KEY }}" >> $GITHUB_ENV
           echo "TF_VAR_did=did:web:staging.indexer.storacha.network" >> $GITHUB_ENV
 
-      - name: Set Production Environment Variables
-        if: startsWith(github.ref, 'refs/tags/')
-        run: |
-          echo "ENV=production" >> $GITHUB_ENV
-          echo "TF_WORKSPACE=prod" >> $GITHUB_ENV
-          echo "TF_VAR_private_key=${{ secrets.PROD_PRIVATE_KEY }}" >> $GITHUB_ENV
-          echo "TF_VAR_did=did:web:indexer.storacha.network" >> $GITHUB_ENV
-          
       - name: Tofu Init
         run: |
           tofu -chdir="deploy/app" init
 
-      - name: Ruild Go Apps    
+      - name: Build Go Apps
         run: |
           touch .env
           make lambdas
@@ -76,3 +72,17 @@ jobs:
         if: github.event_name != 'pull_request'
         run: |
           tofu -chdir="deploy/app" apply -input=false --auto-approve
+
+      # deploy to prod on new releases
+      - name: Set Production Environment Variables
+        if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' }}
+        run: |
+          echo "ENV=production" >> $GITHUB_ENV
+          echo "TF_WORKSPACE=prod" >> $GITHUB_ENV
+          echo "TF_VAR_private_key=${{ secrets.PROD_PRIVATE_KEY }}" >> $GITHUB_ENV
+          echo "TF_VAR_did=did:web:indexer.storacha.network" >> $GITHUB_ENV
+
+      - name: Deploy to prod
+        if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' }}
+        run: |
+          tofu -chdir="deploy/app" apply -input=false --auto-approve
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -1,3 +1,5 @@
+version: 2
+
 builds:
   - main: ./cmd
     binary: indexing-service
@@ -31,4 +33,4 @@ archives:
 release:
   mode: keep-existing
 changelog:
-  skip: true
+  disable: true
diff --git a/deploy/app/gateway.tf b/deploy/app/gateway.tf
@@ -1,3 +1,7 @@
+locals {
+    domain_name = terraform.workspace == "prod" ? "${var.app}.storacha.network" : "${terraform.workspace}.${var.app}.storacha.network"
+}
+
 resource "aws_apigatewayv2_api" "api" {
   name        = "${terraform.workspace}-${var.app}-api"
   description = "${terraform.workspace} ${var.app} API Gateway"
@@ -67,13 +71,14 @@ resource "aws_apigatewayv2_integration" "postclaims" {
 }
 
 resource "aws_apigatewayv2_deployment" "deployment" {
-  depends_on = [aws_apigatewayv2_integration.getclaims, aws_apigatewayv2_integration.getroot, aws_apigatewayv2_integration.postclaims]
+  depends_on = [aws_apigatewayv2_integration.getclaim, aws_apigatewayv2_integration.getclaims, aws_apigatewayv2_integration.getroot, aws_apigatewayv2_integration.postclaims]
   triggers = {
     redeployment = sha1(join(",", [
       jsonencode(aws_apigatewayv2_integration.getclaim),
-      jsonencode(aws_apigatewayv2_integration.postclaims),
       jsonencode(aws_apigatewayv2_integration.getclaims),
+      jsonencode(aws_apigatewayv2_integration.postclaims),
       jsonencode(aws_apigatewayv2_integration.getroot),
+      jsonencode(aws_apigatewayv2_route.getclaim),
       jsonencode(aws_apigatewayv2_route.getclaims),
       jsonencode(aws_apigatewayv2_route.getroot),
       jsonencode(aws_apigatewayv2_route.postclaims),
@@ -97,7 +102,7 @@ data "terraform_remote_state" "shared" {
 }
 
 resource "aws_acm_certificate" "cert" {
-  domain_name       = terraform.workspace == "prod" ? "${var.app}.storacha.network" : "${terraform.workspace}.${var.app}.storacha.network"
+  domain_name       = local.domain_name
   validation_method = "DNS"
 
   lifecycle {
@@ -119,7 +124,7 @@ resource "aws_acm_certificate_validation" "cert" {
   validation_record_fqdns = [ aws_route53_record.cert_validation.fqdn ]
 }
 resource "aws_apigatewayv2_domain_name" "custom_domain" {
-  domain_name = "${terraform.workspace}.${var.app}.storacha.network"
+  domain_name = local.domain_name
 
   domain_name_configuration {
     certificate_arn = aws_acm_certificate_validation.cert.certificate_arn
@@ -130,20 +135,22 @@ resource "aws_apigatewayv2_domain_name" "custom_domain" {
 resource "aws_apigatewayv2_stage" "stage" {
   api_id = aws_apigatewayv2_api.api.id
   name   = "$default"
+  deployment_id = aws_apigatewayv2_deployment.deployment.id
+
   lifecycle {
     create_before_destroy = true
   }
 }
 
 resource "aws_apigatewayv2_api_mapping" "api_mapping" {
   api_id      = aws_apigatewayv2_api.api.id
-  stage  = aws_apigatewayv2_stage.stage.id
+  stage       = aws_apigatewayv2_stage.stage.id
   domain_name = aws_apigatewayv2_domain_name.custom_domain.id
 }
 
 resource "aws_route53_record" "api_gateway" {
   zone_id = data.terraform_remote_state.shared.outputs.primary_zone.zone_id
-  name    = "${terraform.workspace}.${var.app}.storacha.network"
+  name    = aws_apigatewayv2_domain_name.custom_domain.domain_name
   type    = "A"
 
   alias {

diff --git a/deploy/app/lambda.tf b/deploy/app/lambda.tf
@@ -72,9 +72,9 @@ resource "aws_lambda_function" "lambda" {
         PUBLIC_URL = "https://${aws_apigatewayv2_domain_name.custom_domain.domain_name}"
         IPNI_STORE_BUCKET_REGIONAL_DOMAIN = aws_s3_bucket.ipni_store_bucket.bucket_regional_domain_name
         CLAIM_STORE_BUCKET_NAME = aws_s3_bucket.claim_store_bucket.bucket
-        LEGACY_CLAIMS_TABLE_NAME = "prod-content-claims-claims-v1"
-        LEGACY_CLAIMS_BUCKET_NAME = "prod-content-claims-bucket-claimsv1bucketefd46802-1mqz6d8o7xw8"
-        LEGACY_BLOCK_INDEX_TABLE_NAME = "${terraform.workspace == "prod" ? "prod" : "staging"}-ep-v1-blocks-cars-position"
+        LEGACY_CLAIMS_TABLE_NAME = data.aws_dynamodb_table.legacy_claims_table.id
+        LEGACY_CLAIMS_BUCKET_NAME = data.aws_s3_bucket.legacy_claims_bucket.id
+        LEGACY_BLOCK_INDEX_TABLE_NAME = data.aws_dynamodb_table.legacy_block_index_table.id
         LEGACY_DATA_BUCKET_URL = "https://carpark-${terraform.workspace == "prod" ? "prod" : "staging"}-0.r2.w3s.link"
         GOLOG_LOG_LEVEL = terraform.workspace == "prod" ? "error" : "debug"
     }

diff --git a/deploy/app/legacyclaims.tf b/deploy/app/legacyclaims.tf
@@ -0,0 +1,86 @@
+variable "legacy_claims_table_name" {
+  description = "The name of the DynamoDB table used by the legacy content-claims service"
+  type = string
+  default = "prod-content-claims-claims-v1"
+}
+
+variable "legacy_claims_bucket_name" {
+  description = "The name of the S3 bucket used by the legacy content-claims service"
+  type = string
+  default = "prod-content-claims-bucket-claimsv1bucketefd46802-1mqz6d8o7xw8"
+}
+
+variable "legacy_block_index_table_name" {
+  description = "The name of the legacy block index DynamoDB table"
+  type = string
+  default = ""
+}
+
+locals {
+    inferred_legacy_block_index_table_name = var.legacy_block_index_table_name != "" ? var.legacy_block_index_table_name : "${terraform.workspace == "prod" ? "prod" : "staging"}-ep-v1-blocks-cars-position"
+}
+
+data "aws_s3_bucket" "legacy_claims_bucket" {
+  bucket = var.legacy_claims_bucket_name
+}
+
+data "aws_dynamodb_table" "legacy_claims_table" {
+  name = var.legacy_claims_table_name
+}
+
+data "aws_dynamodb_table" "legacy_block_index_table" {
+  name = local.inferred_legacy_block_index_table_name
+}
+
+data "aws_iam_policy_document" "lambda_legacy_dynamodb_query_document" {
+  statement {
+    actions = [
+      "dynamodb:Query",
+    ]
+    resources = [
+      data.aws_dynamodb_table.legacy_claims_table.arn,
+      data.aws_dynamodb_table.legacy_block_index_table.arn
+    ]
+  }
+}
+
+resource "aws_iam_policy" "lambda_legacy_dynamodb_query" {
+  name        = "${terraform.workspace}-${var.app}-lambda-legacy-dynamodb-query"
+  description = "This policy will be used by the lambda to query data from legacy DynamoDB tables"
+  policy      = data.aws_iam_policy_document.lambda_legacy_dynamodb_query_document.json
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_legacy_dynamodb_query" {
+  role       = aws_iam_role.lambda_exec.name
+  policy_arn = aws_iam_policy.lambda_legacy_dynamodb_query.arn
+}
+
+data "aws_iam_policy_document" "lambda_legacy_s3_get_document" {
+  statement {
+    actions = [
+      "s3:GetObject",
+    ]
+    resources = [
+      "${data.aws_s3_bucket.legacy_claims_bucket.arn}/*"
+    ]
+  }
+  statement {
+    actions = [
+      "s3:ListBucket","s3:GetBucketLocation"
+    ]
+    resources = [
+      data.aws_s3_bucket.legacy_claims_bucket.arn
+    ]
+  }
+}
+
+resource "aws_iam_policy" "lambda_legacy_s3_get" {
+  name        = "${terraform.workspace}-${var.app}-lambda_legacy_s3_get"
+  description = "This policy will be used by the lambda to get objects from legacy S3 buckets"
+  policy      = data.aws_iam_policy_document.lambda_legacy_s3_get_document.json
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_legacy_s3_get" {
+  role       = aws_iam_role.lambda_exec.name
+  policy_arn = aws_iam_policy.lambda_legacy_s3_get.arn
+}
diff --git a/deploy/app/main.tf b/deploy/app/main.tf
@@ -24,7 +24,7 @@ provider "aws" {
       "Environment" = terraform.workspace
       "ManagedBy"   = "OpenTofu"
       Owner         = "storacha"
-      Team          = "Storacha Engineer"
+      Team          = "Storacha Engineering"
       Organization  = "Storacha"
       Project       = "${var.app}"
     }

diff --git a/deploy/shared/main.tf b/deploy/shared/main.tf
@@ -21,7 +21,7 @@ provider "aws" {
       "Environment" = terraform.workspace
       "ManagedBy"   = "OpenTofu"
       Owner         = "storacha"
-      Team          = "Storacha Engineer"
+      Team          = "Storacha Engineering"
       Organization  = "Storacha"
       Project       = "${var.app}"
     }

diff --git a/go.mod b/go.mod
@@ -29,7 +29,7 @@ require (
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/redis/go-redis/v9 v9.6.1
 	github.com/storacha/go-capabilities v0.0.0-20241030160329-4cf19ffc732d
-	github.com/storacha/go-metadata v0.0.0-20241031171938-8f3343b7ce5a
+	github.com/storacha/go-metadata v0.0.0-20241216142904-a60e20043cef
 	github.com/storacha/go-ucanto v0.1.1-0.20241028163940-34de8cd912bb
 	github.com/storacha/ipni-publisher v0.0.0-20241112152400-07a540928427
 	github.com/stretchr/testify v1.10.0

diff --git a/go.sum b/go.sum
@@ -651,8 +651,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/storacha/go-capabilities v0.0.0-20241030160329-4cf19ffc732d h1:zcRQSf/3GY/R/wC1W0DKxTGUacsZuwIwhGZH63XWMDE=
 github.com/storacha/go-capabilities v0.0.0-20241030160329-4cf19ffc732d/go.mod h1:vHqUkLGC11Fhio/2qBJkK8MmA4q1t7DaRVdTIo0UHhE=
-github.com/storacha/go-metadata v0.0.0-20241031171938-8f3343b7ce5a h1:c1eCKRLOoNw1oCk7fOUgNArvJlQZYai1QgZOHHAubL4=
-github.com/storacha/go-metadata v0.0.0-20241031171938-8f3343b7ce5a/go.mod h1:HKKrwNQRGo731eC1ourP1ISRBR/Q9CYvbtpSz9yAu6o=
+github.com/storacha/go-metadata v0.0.0-20241216142904-a60e20043cef h1:dWRrteSgq7yIYtKagW/Myk9EZEZqQPk9vukV6qdB8KA=
+github.com/storacha/go-metadata v0.0.0-20241216142904-a60e20043cef/go.mod h1:7iJTSo9FugQ3EPjkR5C4sTHfNv8pxSwGAIbHlFhIrx0=
 github.com/storacha/go-ucanto v0.1.1-0.20241028163940-34de8cd912bb h1:lFwFtMjgt162ot9pnu230haLjRQ1rqwJOIAyywOqAX8=
 github.com/storacha/go-ucanto v0.1.1-0.20241028163940-34de8cd912bb/go.mod h1:7ba9jAgqmwlF/JfyFUQcGV07uiYNlmJNu8qH4hHtrJk=
 github.com/storacha/ipni-publisher v0.0.0-20241112152400-07a540928427 h1:RoP1oLKNH4OXA37h4XgZIPSgFyyRuP3vM4DLbzjc2hk=

diff --git a/pkg/construct/construct.go b/pkg/construct/construct.go
@@ -349,7 +349,7 @@ func Construct(sc ServiceConfig, opts ...Option) (Service, error) {
 			return nil, fmt.Errorf("creating legacy claims store: %w", err)
 		}
 	} else {
-		legacyClaims = providerindex.NewNotFoundLegacyClaimsFinder()
+		legacyClaims = providerindex.NewNoResultsLegacyClaimsFinder()
 	}
 
 	providerIndex := providerindex.New(providersCache, findClient, publisher, legacyClaims)

diff --git a/pkg/internal/testutil/gen.go b/pkg/internal/testutil/gen.go
@@ -13,12 +13,14 @@ import (
 	"github.com/ipld/go-ipld-prime/datamodel"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	"github.com/ipni/go-libipni/find/model"
+	ipnimeta "github.com/ipni/go-libipni/metadata"
 	crypto "github.com/libp2p/go-libp2p/core/crypto"
 	peer "github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
 	mh "github.com/multiformats/go-multihash"
 	"github.com/storacha/go-capabilities/pkg/assert"
+	"github.com/storacha/go-metadata"
 	"github.com/storacha/go-ucanto/core/car"
 	"github.com/storacha/go-ucanto/core/delegation"
 	"github.com/storacha/go-ucanto/core/ipld/block"
@@ -172,6 +174,28 @@ func RandomProviderResult() model.ProviderResult {
 	}
 }
 
+func RandomBitswapProviderResult() model.ProviderResult {
+	pr := RandomProviderResult()
+	bitswapMeta, _ := ipnimeta.Bitswap{}.MarshalBinary()
+	pr.Metadata = bitswapMeta
+	return pr
+}
+
+func RandomLocationCommitmentProviderResult() model.ProviderResult {
+	shard := RandomCID().(cidlink.Link).Cid
+	locationMeta := metadata.LocationCommitmentMetadata{
+		Shard:      &shard,
+		Range:      &metadata.Range{Offset: 128},
+		Expiration: 0,
+		Claim:      RandomCID().(cidlink.Link).Cid,
+	}
+	metaBytes, _ := locationMeta.MarshalBinary()
+
+	pr := RandomProviderResult()
+	pr.Metadata = metaBytes
+	return pr
+}
+
 func RandomShardedDagIndexView(size int) (mh.Multihash, blobindex.ShardedDagIndexView) {
 	root, digest, bytes := RandomCAR(size)
 	shard, err := blobindex.FromShardArchives(root, [][]byte{bytes})

diff --git a/pkg/providerresults/providerresults.go b/pkg/providerresults/providerresults.go
@@ -33,6 +33,11 @@ func init() {
 }
 
 func bytesToPeerID(data []byte) (interface{}, error) {
+	if len(data) == 0 {
+		emptyID := peer.ID("")
+		return &emptyID, nil
+	}
+
 	id, err := peer.IDFromBytes(data)
 	return &id, err
 }

diff --git a/pkg/providerresults/providerresults_test.go b/pkg/providerresults/providerresults_test.go
@@ -56,3 +56,35 @@ func TestProviderResults__Equals(t *testing.T) {
 		})
 	}
 }
+
+func TestSerialization(t *testing.T) {
+	randomResult := testutil.RandomProviderResult()
+
+	testCases := []struct {
+		name       string
+		testResult model.ProviderResult
+	}{
+		{
+			name:       "random result",
+			testResult: randomResult,
+		},
+		{
+			name: "empty peer ID",
+			testResult: model.ProviderResult{
+				ContextID: randomResult.ContextID,
+				Metadata:  randomResult.Metadata,
+				Provider: &peer.AddrInfo{
+					ID:    "",
+					Addrs: randomResult.Provider.Addrs,
+				},
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			marshalled := testutil.Must(providerresults.MarshalCBOR([]model.ProviderResult{tc.testResult}))(t)
+			unmarshalled := testutil.Must(providerresults.UnmarshalCBOR(marshalled))(t)
+			require.Equal(t, tc.testResult, unmarshalled[0])
+		})
+	}
+}