Use OpenTelemetry OCI S3 bucket for coredump testing

.github/workflows/unit-test-on-pull-request.yml

@@ -56,6 +56,14 @@ jobs:
         uses: actions/checkout@v4
       - name: Set up environment
         uses: ./.github/workflows/env
+      - name: Cache coredump modules
+        uses: actions/cache@v4
+        with:
+          path: tools/coredump/modulecache
+          key: coredumps-${{ matrix.target_arch }}-${{ hashFiles('tools/coredump/testdata/*/*.json') }}
+          restore-keys: |
+            coredumps-${{ matrix.target_arch }}
+            coredumps-
       - name: Tests
         run: make test TARGET_ARCH=${{ matrix.target_arch }}
 

tools/coredump/coredump_test.go

@@ -3,33 +3,37 @@
 
 package main
 
-// NOTE: temporarily disabled until we figured out how to best do this without S3 in the OTel env
-
-//nolint:gocritic
-//func TestCoreDumps(t *testing.T) {
-//	cases, err := findTestCases(true)
-//	require.NoError(t, err)
-//	require.NotEmpty(t, cases)
-//
-//	store, err := initModuleStore()
-//	require.NoError(t, err)
-//
-//	for _, filename := range cases {
-//		filename := filename
-//		t.Run(filename, func(t *testing.T) {
-//			testCase, err := readTestCase(filename)
-//			require.NoError(t, err)
-//
-//			ctx := context.Background()
-//
-//			core, err := OpenStoreCoredump(store, testCase.CoredumpRef, testCase.Modules)
-//			require.NoError(t, err)
-//			defer core.Close()
-//
-//			data, err := ExtractTraces(ctx, core, false, nil)
-//
-//			require.NoError(t, err)
-//			require.Equal(t, testCase.Threads, data)
-//		})
-//	}
-//}
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCoreDumps(t *testing.T) {
+	cases, err := findTestCases(true)
+	require.NoError(t, err)
+	require.NotEmpty(t, cases)
+
+	store, err := initModuleStore()
+	require.NoError(t, err)
+
+	for _, filename := range cases {
+		filename := filename
+		t.Run(filename, func(t *testing.T) {
+			testCase, err := readTestCase(filename)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+
+			core, err := OpenStoreCoredump(store, testCase.CoredumpRef, testCase.Modules)
+			require.NoError(t, err)
+			defer core.Close()
+
+			data, err := ExtractTraces(ctx, core, false, nil)
+
+			require.NoError(t, err)
+			require.Equal(t, testCase.Threads, data)
+		})
+	}
+}

tools/coredump/main.go

@@ -11,8 +11,10 @@ import (
 	"context"
 	"errors"
 	"flag"
+	"fmt"
 	"os"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 
@@ -21,8 +23,19 @@ import (
 	"go.opentelemetry.io/ebpf-profiler/tools/coredump/modulestore"
 )
 
+// moduleStoreRegion defines the S3 bucket OCI region.
+const moduleStoreRegion = "us-sanjose-1"
+
+// moduleStoreObjectNamespace defines the S3 bucket OCI object name space.
+const moduleStoreObjectNamespace = "axtwf1hkrwcy"
+
+// modulePublicReadUrl defines the S3 bucket OCI public read only base path.
+//
+//nolint:lll
+const modulePublicReadURL = "sm-wftyyzHJkBghWeexmK1o5ArimNwZC-5eBej5Lx4e46sLVHtO_y7Zf7FZgoIu_/n/axtwf1hkrwcy"
+
 // moduleStoreS3Bucket defines the S3 bucket used for the module store.
-const moduleStoreS3Bucket = "optimyze-proc-mem-testdata"
+const moduleStoreS3Bucket = "ebpf-profiling-coredumps"
 
 func main() {
 	log.SetReportCaller(false)
@@ -59,10 +72,20 @@ func main() {
 }
 
 func initModuleStore() (*modulestore.Store, error) {
+	publicReadURL := fmt.Sprintf("https://%s.objectstorage.%s.oci.customer-oci.com/p/%s/b/%s/o/",
+		moduleStoreObjectNamespace, moduleStoreRegion, modulePublicReadURL, moduleStoreS3Bucket)
+
 	cfg, err := awsconfig.LoadDefaultConfig(context.Background())
 	if err != nil {
 		return nil, err
 	}
-	s3Client := s3.NewFromConfig(cfg)
-	return modulestore.New(s3Client, moduleStoreS3Bucket, "modulecache")
+
+	s3Client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		baseEndpoint := fmt.Sprintf("https://%s.compat.objectstorage.%s.oraclecloud.com/",
+			moduleStoreObjectNamespace, moduleStoreRegion)
+		o.Region = moduleStoreRegion
+		o.BaseEndpoint = aws.String(baseEndpoint)
+		o.UsePathStyle = true
+	})
+	return modulestore.New(s3Client, publicReadURL, moduleStoreS3Bucket, "modulecache")
 }

tools/coredump/modulestore/store.go

@@ -11,13 +11,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path"
 	"strings"
 	"syscall"
 	"time"
 
-	s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 
@@ -53,18 +53,27 @@ const (
 // at the same time, also when created within multiple different applications.
 type Store struct {
 	s3client       *s3.Client
+	httpClient     *http.Client
+	publicReadURL  string
 	bucket         string
 	localCachePath string
 }
 
 // New creates a new module storage. The modules present in the local cache are inspected and a
 // full index of the modules in the remote S3 bucket is retrieved and cached as well.
-func New(s3client *s3.Client, s3Bucket, localCachePath string) (*Store, error) {
+func New(s3client *s3.Client, publicReadURL, s3Bucket, localCachePath string) (*Store, error) {
 	if err := os.MkdirAll(localCachePath, 0o750); err != nil {
 		return nil, err
 	}
+	tr := &http.Transport{
+		MaxIdleConns:       2,
+		IdleConnTimeout:    30 * time.Second,
+		DisableCompression: true,
+	}
 	return &Store{
 		s3client:       s3client,
+		httpClient:     &http.Client{Transport: tr},
+		publicReadURL:  publicReadURL,
 		bucket:         s3Bucket,
 		localCachePath: localCachePath,
 	}, nil
@@ -179,7 +188,7 @@ func (store *Store) UploadModule(id ID) error {
 		return fmt.Errorf("failed to check whether the module exists locally: %w", err)
 	}
 	if !present {
-		return fmt.Errorf("the given module `%s` isn't present locally", id)
+		return fmt.Errorf("the given module `%x` isn't present locally", id)
 	}
 
 	localPath := store.makeLocalPath(id)
@@ -425,18 +434,13 @@ func (store *Store) ensurePresentLocally(id ID) (string, error) {
 	defer file.Close()
 
 	moduleKey := makeS3Key(id)
-	req := &s3.GetObjectInput{
-		Bucket: &store.bucket,
-		Key:    &moduleKey,
-	}
-
-	downloader := s3manager.NewDownloader(store.s3client)
-	_, err = downloader.Download(context.TODO(), file, req)
+	resp, err := http.Get(store.publicReadURL + moduleKey)
 	if err != nil {
-		if isErrNoSuchKey(err) {
-			return "", errors.New("module doesn't exist in remote storage")
-		}
-		return "", fmt.Errorf("failed to download file from S3: %w", err)
+		return "", fmt.Errorf("failed to request file: %w", err)
+	}
+	defer resp.Body.Close()
+	if _, err = io.Copy(file, resp.Body); err != nil {
+		return "", fmt.Errorf("failed to receive file: %w", err)
 	}
 
 	if err = commitTempFile(file, localPath); err != nil {

tools/coredump/testdata/amd64/graalvm-native.json

@@ -1,5 +1,5 @@
 {
-  "coredump-ref": "a9c4a72d50d0c9fa133b3d39756405ba431cfe329e39b3f9dd28ab9cbd256076",
+  "coredump-ref": "e7fa60d8b9eddd1a29c2aeea9b21ef625fa3ad1559e37a5433958620c9841f1f",
   "threads": [
     {
       "lwp": 3327417,

tools/coredump/testdata/amd64/perl534.220234.json

@@ -1,5 +1,5 @@
 {
-  "coredump-ref": "d7fa9bf0effe15fd628f1cfa50a2e2d22c574f2c3e8e794f248b34c6b0e6fc66",
+  "coredump-ref": "682732701a4de336b4e9fcfd25caaf963d918ce8b5e554afeb001e63d956bd86",
   "threads": [
     {
       "lwp": 220234,

tools/coredump/testdata/amd64/python310.stringbench.20086.json

@@ -1,5 +1,5 @@
 {
-  "coredump-ref": "94a9c5cd6295d08274ff0c4f513a0157b220675c252753e2bd7598f3d4017dde",
+  "coredump-ref": "caf74bc49055cd6959fe96a6c14b19da025254b2351293a1b9d1b9165d78074e",
   "threads": [
     {
       "lwp": 20086,

tools/coredump/testdata/amd64/python311.50282.json

@@ -1,5 +1,5 @@
 {
-  "coredump-ref": "07b722f7560748f27cb9f7bc5822a3054f817e60c44a36b77f51a1e9ad3875bc",
+  "coredump-ref": "2d15786c1581bb1e069be0d12cd50b3de99532951d85668a1961c6a35cdd3542",
   "threads": [
     {
       "lwp": 50282,

tools/coredump/testdata/amd64/ruby30.253432.json

@@ -1,5 +1,5 @@
 {
-  "coredump-ref": "ff1183a2679a72cdd32b3963af31bbf0aa3067c6e713cd02e6722015673e0633",
+  "coredump-ref": "1b496ac7c0747b896e6c5ef5720db01d900a224a529eb74d5ce0c787a7e4bcd1",
   "threads": [
     {
       "lwp": 253432,