Add codex library integration

Author: 2-towns

.gitignore

@@ -21,3 +21,5 @@ coverage*.cov
 
 # Logs
 *.log
+
+libs

.vscode/settings.json

@@ -1,6 +1,13 @@
 {
   "go.testTags": "codex_integration",
   "gopls": {
-    "buildFlags": ["-tags=codex_integration"]
+    "buildFlags": [
+      "-tags=integration"
+    ]
+  },
+  "go.toolsEnvVars": {
+    "CGO_ENABLED": "1",
+    "CGO_CFLAGS": "-I${workspaceFolder}/libs",
+    "CGO_LDFLAGS": "-L${workspaceFolder}/libs -lcodex -Wl,-rpath,${workspaceFolder}/libs"
   }
-}
+}

Makefile

@@ -0,0 +1,46 @@
+# Destination folder for the downloaded libraries
+LIBS_DIR := $(abspath ./libs)
+
+# Flags for CGO to find the headers and the shared library
+UNAME_S := $(shell uname -s)
+CGO_CFLAGS  := -I$(LIBS_DIR)
+CGO_LDFLAGS := -L$(LIBS_DIR) -lcodex -Wl,-rpath,$(LIBS_DIR)
+
+ifeq ($(OS),Windows_NT)
+  BIN_NAME := codex-go.exe
+else
+  BIN_NAME := codex-go
+endif
+
+# Configuration for fetching the right binary
+OS ?= "linux"
+ARCH ?= "amd64"
+VERSION ?= "v0.0.22"
+DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(VERSION)/codex-${OS}-${ARCH}.zip"		
+
+fetch: 
+	@echo "Fetching libcodex from GitHub Actions from: ${DOWNLOAD_URL}"
+	curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${OS}-${ARCH}.zip ${DOWNLOAD_URL}
+	unzip -o -qq $(LIBS_DIR)/codex-${OS}-${ARCH}.zip -d $(LIBS_DIR)
+	rm -f $(LIBS_DIR)/*.zip
+
+build:
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o $(BIN_NAME) main.go
+
+build-upload:
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-upload ./cmd/upload
+
+build-download:
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-download ./cmd/download
+
+test:
+	@echo "Running unit tests..."
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -v ./communities
+
+test-integration:
+	@echo "Running tests..."
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go  test -v -tags=codex_integration ./communities -run Integration -timeout 15s
+
+clean:
+	rm -f $(BIN_NAME)
+	rm -Rf $(LIBS_DIR)/*

README.md

@@ -10,43 +10,30 @@ A lightweight Go client utility for interacting with Codex client.
 
 We will be running codex client, and then use a small testing utility to check if the low level abstraction - CodexClient - correctly uploads and downloads the content.
 
-### Running CodexClient
+### Integration Codex library
 
-I often remove some logging noise, by slightly changing the build
-params in `build.nims` (nim-codex):
+You need to download the library file by using: 
 
-```nim
-task codex, "build codex binary":
-  buildBinary "codex",
-    # params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
-    params =
-      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE"
-```
-
-You see a slightly more selective `params` in the `codex` task.
-
-To run the client I use the following command:
-
-```bash
-./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE
+```sh
+make fetch
 ```
 
 ### Building codex-upload and codex-download utilities
 
 Use the following command to build the `codex-upload` and `codex-download` utilities:
 
 ```bash
-go build -o bin/codex-upload ./cmd/upload
-go build -o bin/codex-download ./cmd/download
+make build-upload
+make build-download
 ```
 ### Uploading content to Codex
 
 Now, using the `codex-upload` utility, we can upload the content to Codex as follows:
 
 ```bash
 ~/code/local/go-codex-client
-❯ ./bin/codex-upload -file test-data.bin -host localhost -port 8001
-Uploading test-data.bin (43 bytes) to Codex at localhost:8001...
+❯ ./bin/codex-upload -file test-data.bin
+Uploading test-data.bin (43 bytes) to Codex
 ✅ Upload successful!
 CID: zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V
 ```
@@ -57,8 +44,8 @@ Now, having the content uploaded to Codex - let's get it back using the `codex-d
 
 ```bash
 ~/code/local/go-codex-client
-❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin -host localhost -port 8001
-Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex at localhost:8001...
+❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin
+Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex...
 ✅ Download successful!
 Saved to: output.bin
 ```
@@ -85,115 +72,23 @@ next section.
 To run all unit tests:
 
 ```bash
-❯ go test -v ./communities -count 1
-```
-
-To be more selective, e.g. in order to run all the tests from 
-`CodexArchiveDownloaderSuite`, run:
-
-```bash
-go test -v ./communities -run CodexArchiveDownloader -count 1
-```
-
-or for an individual test from that suite:
-
-```bash
-go test -v ./communities -run TestCodexArchiveDownloaderSuite/TestCancellationDuringPolling -count 1
-```
-
-You can also use `gotestsum` to run the tests (you may need to install it first, e.g. `go install gotest.tools/[email protected]`):
-
-```bash
-gotestsum --packages="./communities" -f testname --rerun-fails -- -count 1
-```
-
-For a more verbose output including logs use `-f standard-verbose`, e.g.:
-
-```bash
-gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -v -count 1
-```
-
-To be more selective, e.g. in order to run all the tests from 
-`CodexArchiveDownloaderSuite`, run:
-
-```bash
-gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArchiveDownloader -count 1
-```
-
-or for an individual test from that suite:
-
-```bash
-gotestsum --packages="./communities" -f testname --rerun-fails -- -run TestCodexArchiveDownloaderSuite/TestCancellationDuringPolling -count 1
-```
-
-Notice, that the `-run` flag accepts a regular expression that matches against the full test path, so you can be more concise in naming if necessary, e.g.:
-
-```bash
-gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArchiveDownloader/Cancellation -count 1
-```
-
-This also applies to native `go test` command.
-
-### Running integration tests
-
-When building Codex client for testing like here, I often remove some logging noise, by slightly changing the build params in `build.nims`:
-
-```nim
-task codex, "build codex binary":
-  buildBinary "codex",
-    # params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
-    params =
-      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE"
-```
-
-You see a slightly more selective `params` in the `codex` task.
-
-To start Codex client, use e.g.:
-
-```bash
-./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE
-```
-
-To run the integration test, use `codex_integration` tag and narrow the scope using `-run Integration`:
-
-```bash
-CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s
-```
-
-This will run all integration tests, including CodexClient integration tests.
-
-To make sure that the test is actually run and not cached, use `count` option:
-
-```bash
-CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s -count 1
-```
-
-To be more specific and only run the tests related to, e.g. index downloader or archive
-downloader you can use:
-
-```bash
-CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexIndexDownloaderIntegration -timeout 15s -count 1
-
-CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexArchiveDownloaderIntegration -timeout 15s -count 1
-```
-
-and then, if you prefer to use `gotestsum`:
-
-```bash
-CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexIndexDownloaderIntegration -v -count 1
-
-CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexArchiveDownloaderIntegration -v -count 1
+❯ make test
+=== RUN   TestUpload_Success
+--- PASS: TestUpload_Success (0.00s)
+=== RUN   TestDownload_Success
+--- PASS: TestDownload_Success (0.00s)
+=== RUN   TestDownloadWithContext_Cancel
+--- PASS: TestDownloadWithContext_Cancel (0.04s)
+PASS
+ok  	go-codex-client/communities	0.044s
 ```
 
-or to run all integration tests (including CodexClient integration tests):
+To run the integration test, use `test-integration`:
 
 ```bash
-CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -v -count 1 -run Integration
+make test-integration
 ```
 
-I prefer to be more selective when running integration tests.
-
-
 ### Regenerating artifacts
 
 Everything you need comes included in the repo. But if you decide to change things,

cmd/download/main.go

@@ -5,14 +5,15 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 
 	"go-codex-client/communities" // Import the local communities package
+
+	"github.com/codex-storage/codex-go-bindings/codex"
 )
 
 func main() {
 	var (
-		host = flag.String("host", "localhost", "Codex host")
-		port = flag.String("port", "8080", "Codex port")
 		cid  = flag.String("cid", "", "CID of the file to download")
 		file = flag.String("file", "downloaded-file.bin", "File to save the downloaded data")
 	)
@@ -24,7 +25,20 @@ func main() {
 	}
 
 	// Create Codex client
-	client := communities.NewCodexClient(*host, *port)
+	client, err := communities.NewCodexClient(codex.Config{
+		LogFormat:      codex.LogFormatNoColors,
+		MetricsEnabled: false,
+		BlockRetries:   5,
+		LogLevel:       "ERROR",
+		DataDir:        path.Join(os.TempDir(), "codex-client-data"),
+	})
+	if err != nil {
+		log.Fatalf("Failed to create CodexClient: %v", err)
+	}
+
+	if err := client.Start(); err != nil {
+		log.Fatalf("Failed to start CodexClient: %v", err)
+	}
 
 	// Create output file
 	outputFile, err := os.Create(*file)
@@ -33,8 +47,6 @@ func main() {
 	}
 	defer outputFile.Close()
 
-	fmt.Printf("Downloading CID %s from Codex at %s:%s...\n", *cid, *host, *port)
-
 	// Download data - pass the io.Writer (outputFile), not the string
 	err = client.Download(*cid, outputFile)
 	if err != nil {
@@ -43,6 +55,13 @@ func main() {
 		log.Fatalf("Download failed: %v", err)
 	}
 
+	if err := client.Stop(); err != nil {
+		log.Printf("Warning: Failed to stop CodexClient: %v", err)
+	}
+	if err := client.Destroy(); err != nil {
+		log.Printf("Warning: Failed to stop CodexClient: %v", err)
+	}
+
 	fmt.Printf("✅ Download successful!\n")
 	fmt.Printf("Saved to: %s\n", *file)
 }

cmd/upload/main.go

@@ -6,14 +6,15 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 
 	"go-codex-client/communities" // Import the local communities package
+
+	"github.com/codex-storage/codex-go-bindings/codex"
 )
 
 func main() {
 	var (
-		host     = flag.String("host", "localhost", "Codex host")
-		port     = flag.String("port", "8080", "Codex port")
 		file     = flag.String("file", "test-data.bin", "File to upload")
 		filename = flag.String("name", "", "Filename to use in upload (defaults to actual filename)")
 	)
@@ -31,15 +32,35 @@ func main() {
 		uploadName = *file
 	}
 
-	fmt.Printf("Uploading %s (%d bytes) to Codex at %s:%s...\n", *file, len(data), *host, *port)
+	fmt.Printf("Uploading %s (%d bytes) to Codex...\n", *file, len(data))
 	// Create Codex client and upload
-	client := communities.NewCodexClient(*host, *port)
+	client, err := communities.NewCodexClient(codex.Config{
+		LogFormat:      codex.LogFormatNoColors,
+		MetricsEnabled: false,
+		BlockRetries:   5,
+		LogLevel:       "ERROR",
+		DataDir:        path.Join(os.TempDir(), "codex-client-data"),
+	})
+	if err != nil {
+		log.Fatalf("Failed to create CodexClient: %v", err)
+	}
+
+	if err := client.Start(); err != nil {
+		log.Fatalf("Failed to start CodexClient: %v", err)
+	}
 
 	cid, err := client.Upload(bytes.NewReader(data), uploadName)
 	if err != nil {
 		log.Fatalf("Upload failed: %v", err)
 	}
 
+	if err := client.Stop(); err != nil {
+		log.Printf("Warning: Failed to stop CodexClient: %v", err)
+	}
+	if err := client.Destroy(); err != nil {
+		log.Printf("Warning: Failed to stop CodexClient: %v", err)
+	}
+
 	fmt.Printf("✅ Upload successful!\n")
 	fmt.Printf("CID: %s\n", cid)
 }

communities/codex_archive_downloader.go

@@ -316,8 +316,8 @@ func (d *CodexArchiveDownloader) triggerSingleArchiveDownload(hash, cid string,
 		return fmt.Errorf("failed to trigger archive download with CID %s: %w", cid, err)
 	}
 
-	if manifest.CID != cid {
-		return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.CID, cid)
+	if manifest.Cid != cid {
+		return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.Cid, cid)
 	}
 
 	return nil