diff --git a/pkg/node/helper.go b/pkg/node/helper.go index c598c3230..b9ccc2740 100644 --- a/pkg/node/helper.go +++ b/pkg/node/helper.go @@ -7,6 +7,8 @@ import ( "encoding/json" "errors" "fmt" + "os" + "path/filepath" "regexp" "sort" "strconv" @@ -24,8 +26,11 @@ import ( "github.com/ava-labs/avalanche-cli/pkg/utils" "github.com/ava-labs/avalanche-cli/pkg/ux" "github.com/ava-labs/avalanche-cli/pkg/vm" + "github.com/ava-labs/avalanche-cli/sdk/network" + "github.com/ava-labs/avalanche-cli/sdk/publicarchive" "github.com/ava-labs/avalanche-network-runner/rpcpb" "github.com/ava-labs/avalanchego/api/info" + "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/set" ) @@ -555,3 +560,75 @@ func GetNodeData(endpoint string) ( "0x" + hex.EncodeToString(proofOfPossession.ProofOfPossession[:]), nil } + +func SeedClusterData( + clusterNetwork models.Network, + rootDir string, + nodeNames []string, +) error { + // only fuji is supported for now + if clusterNetwork.Kind != models.Fuji { + return fmt.Errorf("unsupported network: %s", clusterNetwork.Name()) + } + network := network.FujiNetwork() + ux.Logger.Info("downloading public archive for network %s", clusterNetwork.Name()) + publicArcDownloader, err := publicarchive.NewDownloader(network, logging.Off) // off as we run inside of the spinner + if err != nil { + return fmt.Errorf("failed to create public archive downloader for network %s: %w", clusterNetwork.Name(), err) + } + + if err := publicArcDownloader.Download(); err != nil { + return fmt.Errorf("failed to download public archive: %w", err) + } + // defer publicArcDownloader.CleanUp() + if path, err := publicArcDownloader.GetDownloadedFilePath(); err != nil { + return fmt.Errorf("failed to get downloaded file path: %w", err) + } else { + ux.Logger.Info("public archive downloaded to %s", path) + } + + wg := sync.WaitGroup{} + mu := sync.Mutex{} + var firstErr error + + for _, nodeName := range nodeNames { + target := filepath.Join(rootDir, nodeName, "db") + ux.Logger.Info("unpacking public archive to %s", target) + + // Skip if target already exists + if _, err := os.Stat(target); err == nil { + ux.Logger.Info("data folder already exists at %s. Skipping...", target) + continue + } + wg.Add(1) + go func(target string) { + defer wg.Done() + + if err := publicArcDownloader.UnpackTo(target); err != nil { + // Capture the first error encountered + mu.Lock() + if firstErr == nil { + firstErr = fmt.Errorf("failed to unpack public archive: %w", err) + _ = CleanUpClusterNodeData(rootDir, nodeNames) + } + mu.Unlock() + } + }(target) + } + wg.Wait() + + if firstErr != nil { + return firstErr + } + ux.Logger.PrintToUser("Public archive unpacked to: %s", rootDir) + return nil +} + +func CleanUpClusterNodeData(rootDir string, nodesNames []string) error { + for _, nodeName := range nodesNames { + if err := os.RemoveAll(filepath.Join(rootDir, nodeName)); err != nil { + return err + } + } + return nil +} diff --git a/pkg/node/local.go b/pkg/node/local.go index ae2e83e7c..b04ebc19e 100644 --- a/pkg/node/local.go +++ b/pkg/node/local.go @@ -336,6 +336,13 @@ func StartLocalNode( } } if network.Kind == models.Fuji { + // disable indexing for fuji + nodeConfig[config.IndexEnabledKey] = false + nodeConfigBytes, err := json.Marshal(nodeConfig) + if err != nil { + return err + } + nodeConfigStr = string(nodeConfigBytes) ux.Logger.PrintToUser(logging.Yellow.Wrap("Warning: Fuji Bootstrapping can take several minutes")) } if err := preLocalChecks(anrSettings, avaGoVersionSetting, useEtnaDevnet, globalNetworkFlags); err != nil { @@ -417,6 +424,14 @@ func StartLocalNode( ux.Logger.PrintToUser("Starting local avalanchego node using root: %s ...", rootDir) spinSession := ux.NewUserSpinner() spinner := spinSession.SpinToUser("Booting Network. Wait until healthy...") + // preseed nodes data from public archive. ignore errors + nodeNames := []string{} + for i := 1; i <= int(numNodes); i++ { + nodeNames = append(nodeNames, fmt.Sprintf("node%d", i)) + } + err := SeedClusterData(network, rootDir, nodeNames) + ux.Logger.Info("seeding public archive data finished with error: %v. Ignored if any", err) + if _, err := cli.Start(ctx, avalancheGoBinPath, anrOpts...); err != nil { ux.SpinFailWithError(spinner, "", err) _ = DestroyLocalNode(app, clusterName) @@ -475,6 +490,9 @@ func UpsizeLocalNode( nodeConfig = map[string]interface{}{} } nodeConfig[config.NetworkAllowPrivateIPsKey] = true + if network.Kind == models.Fuji { + nodeConfig[config.IndexEnabledKey] = false // disable index for Fuji + } nodeConfigBytes, err := json.Marshal(nodeConfig) if err != nil { return "", err @@ -556,6 +574,8 @@ func UpsizeLocalNode( spinSession := ux.NewUserSpinner() spinner := spinSession.SpinToUser("Creating new node with name %s on local machine", newNodeName) + err = SeedClusterData(network, rootDir, []string{newNodeName}) + ux.Logger.Info("seeding public archive data finished with error: %v. Ignored if any", err) // add new local node if _, err := cli.AddNode(ctx, newNodeName, avalancheGoBinPath, anrOpts...); err != nil { ux.SpinFailWithError(spinner, "", err) diff --git a/sdk/constants/constants.go b/sdk/constants/constants.go index d985e24b1..ed0ddd382 100644 --- a/sdk/constants/constants.go +++ b/sdk/constants/constants.go @@ -10,5 +10,6 @@ const ( APIRequestLargeTimeout = 2 * time.Minute // node - WriteReadUserOnlyPerms = 0o600 + WriteReadUserOnlyPerms = 0o600 + WriteReadUserOnlyDirPerms = 0o700 ) diff --git a/sdk/publicarchive/README.md b/sdk/publicarchive/README.md new file mode 100644 index 000000000..84675ee04 --- /dev/null +++ b/sdk/publicarchive/README.md @@ -0,0 +1,55 @@ +# Public Archive Downloader SDK + +This Go package provides a utility to download and extract tar archives from public URLs. It's tailored for downloading Avalanche network archives but can be adapted for other use cases. + + +## Features + +* Downloads files from predefined URLs. +* Tracks download progress and logs status updates. +* Safely unpacks .tar archives to a target directory. +* Includes security checks to prevent path traversal and manage large files. + +## Usage example + +``` +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved +// See the file LICENSE for licensing terms. + +``` +package main + +import ( + "fmt" + "os" + + "github.com/ava-labs/avalanche-cli/sdk/network" + "github.com/ava-labs/avalanchego/utils/constants" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/your-repo-name/publicarchive" +) + +func main() { + // Initialize the downloader + downloader, err := publicarchive.NewDownloader(network.FujiNetwork(), logging.Debug) + if err != nil { + fmt.Printf("Failed to create downloader: %v\n", err) + os.Exit(1) + } + + // Start downloading + if err := downloader.Download(); err != nil { + fmt.Printf("Download failed: %v\n", err) + os.Exit(1) + } + + // Specify the target directory for unpacking + targetDir := "./extracted_files" + if err := downloader.UnpackTo(targetDir); err != nil { + fmt.Printf("Failed to unpack archive: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Files successfully unpacked to %s\n", targetDir) +} +``` diff --git a/sdk/publicarchive/downloader.go b/sdk/publicarchive/downloader.go new file mode 100644 index 000000000..d8e88f3cf --- /dev/null +++ b/sdk/publicarchive/downloader.go @@ -0,0 +1,215 @@ +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package publicarchive + +import ( + "archive/tar" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/ava-labs/avalanche-cli/sdk/network" + "github.com/ava-labs/avalanchego/utils/constants" + "github.com/ava-labs/avalanchego/utils/logging" + "github.com/cavaliergopher/grab/v3" + "go.uber.org/zap" + + sdkConstants "github.com/ava-labs/avalanche-cli/sdk/constants" +) + +const ( + updateInterval = 500 * time.Millisecond + maxFileSize = 10 * 1024 * 1024 * 1024 // 10GB per file + // public archive + PChainArchiveFuji = "https://avalanchego-public-database.avax-test.network/testnet/p-chain/avalanchego/data-tar/latest.tar" +) + +type Getter struct { + client *grab.Client + request *grab.Request + size int64 + bytesComplete int64 + mutex *sync.RWMutex +} + +type Downloader struct { + getter Getter + logger logging.Logger + currentOp *sync.Mutex +} + +// newGetter returns a new Getter +func newGetter(endpoint string, target string) (Getter, error) { + if request, err := grab.NewRequest(target, endpoint); err != nil { + return Getter{}, err + } else { + return Getter{ + client: grab.NewClient(), + request: request, + size: 0, + bytesComplete: 0, + mutex: &sync.RWMutex{}, + }, nil + } +} + +// NewDownloader returns a new Downloader +// network: the network to download from ( fuji or mainnet). todo: add mainnet support +// target: the path to download to +// logLevel: the log level +func NewDownloader( + network network.Network, + logLevel logging.Level, +) (Downloader, error) { + tmpFile, err := os.CreateTemp("", "avalanche-cli-public-archive-*") + if err != nil { + return Downloader{}, err + } + + switch network.ID { + case constants.FujiID: + if getter, err := newGetter(PChainArchiveFuji, tmpFile.Name()); err != nil { + return Downloader{}, err + } else { + return Downloader{ + getter: getter, + logger: logging.NewLogger("public-archive-downloader", logging.NewWrappedCore(logLevel, os.Stdout, logging.JSON.ConsoleEncoder())), + currentOp: &sync.Mutex{}, + }, nil + } + default: + return Downloader{}, fmt.Errorf("unsupported network ID: %d", network.ID) + } +} + +func (d Downloader) Download() error { + d.logger.Info("Download started from", zap.String("url", d.getter.request.URL().String())) + + d.currentOp.Lock() + defer d.currentOp.Unlock() + + resp := d.getter.client.Do(d.getter.request) + d.setDownloadSize(resp.Size()) + d.logger.Debug("Download response received", + zap.String("status", resp.HTTPResponse.Status)) + t := time.NewTicker(updateInterval) + defer t.Stop() + + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-t.C: + d.setBytesComplete(resp.BytesComplete()) + d.logger.Info("Download progress", + zap.Int64("bytesComplete", d.GetBytesComplete()), + zap.Int64("size", d.GetDownloadSize())) + case <-resp.Done: + return + } + } + }() + <-resp.Done // Wait for the download to finish + t.Stop() // Stop the ticker + <-done + + // check for errors + if err := resp.Err(); err != nil { + d.logger.Error("Download failed", zap.Error(err)) + return err + } + + d.logger.Info("Download saved to", zap.String("path", d.getter.request.Filename)) + return nil +} + +func (d Downloader) UnpackTo(targetDir string) error { + d.currentOp.Lock() + defer d.currentOp.Unlock() + // prepare destination path + if err := os.MkdirAll(targetDir, sdkConstants.WriteReadUserOnlyDirPerms); err != nil { + d.logger.Error("Failed to create target directory", zap.Error(err)) + return err + } + tarFile, err := os.Open(d.getter.request.Filename) + if err != nil { + d.logger.Error("Failed to open tar file", zap.Error(err)) + return fmt.Errorf("failed to open tar file: %w", err) + } + defer tarFile.Close() + + tarReader := tar.NewReader(io.LimitReader(tarFile, maxFileSize)) + extractedSize := int64(0) + for { + header, err := tarReader.Next() + if err == io.EOF { + d.logger.Debug("End of archive reached") + break // End of archive + } + if err != nil { + d.logger.Error("Failed to read tar archive", zap.Error(err)) + return fmt.Errorf("error reading tar archive: %w", err) + } + + relPath, err := filepath.Rel(targetDir, filepath.Join(targetDir, filepath.Clean(header.Name))) + if err != nil || strings.HasPrefix(relPath, "..") { + d.logger.Error("Invalid file path", zap.String("path", header.Name)) + return fmt.Errorf("invalid file path: %s", header.Name) + } + targetPath := filepath.Join(targetDir, relPath) + + // security checks + if extractedSize+header.Size > maxFileSize { + d.logger.Error("File too large", zap.String("path", header.Name), zap.Int64("size", header.Size)) + return fmt.Errorf("file too large: %s", header.Name) + } + if strings.Contains(header.Name, "..") { + d.logger.Error("Invalid file path", zap.String("path", header.Name)) + return fmt.Errorf("invalid file path: %s", header.Name) + } + // end of security checks + + switch header.Typeflag { + case tar.TypeDir: + d.logger.Debug("Creating directory", zap.String("path", targetPath)) + if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil { + d.logger.Error("Failed to create directory", zap.Error(err)) + return fmt.Errorf("failed to create directory: %w", err) + } + case tar.TypeReg: + d.logger.Debug("Ensure parent directory exists for ", zap.String("path", targetPath)) + if err := os.MkdirAll(filepath.Dir(targetPath), os.FileMode(0o755)); err != nil { + d.logger.Error("Failed to create parent directory for file", zap.Error(err)) + return fmt.Errorf("failed to create parent directory for file: %w", err) + } + d.logger.Debug("Creating file", zap.String("path", targetPath)) + outFile, err := os.Create(targetPath) + if err != nil { + d.logger.Error("Failed to create file", zap.Error(err)) + return fmt.Errorf("failed to create file: %w", err) + } + defer outFile.Close() + copied, err := io.CopyN(outFile, tarReader, header.Size) + if err != nil { + d.logger.Error("Failed to write file", zap.Error(err)) + return fmt.Errorf("failed to write file: %w", err) + } + if copied < header.Size { + d.logger.Error("Incomplete file write", zap.String("path", targetPath)) + return fmt.Errorf("incomplete file write for %s", targetPath) + } + extractedSize += header.Size + d.logger.Debug("Written bytes", zap.Int64("bytes", extractedSize)) + default: + d.logger.Debug("Skipping file", zap.String("path", targetPath)) + } + } + d.logger.Info("Download unpacked to", zap.String("path", targetDir)) + return nil +} diff --git a/sdk/publicarchive/downloader_test.go b/sdk/publicarchive/downloader_test.go new file mode 100644 index 000000000..6301ee921 --- /dev/null +++ b/sdk/publicarchive/downloader_test.go @@ -0,0 +1,177 @@ +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package publicarchive + +import ( + "archive/tar" + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/cavaliergopher/grab/v3" + "github.com/stretchr/testify/require" + + "github.com/ava-labs/avalanche-cli/sdk/network" + "github.com/ava-labs/avalanchego/utils/constants" + "github.com/ava-labs/avalanchego/utils/logging" +) + +func TestNewGetter(t *testing.T) { + endpoint := "http://example.com/file.tar" + target := "/tmp/file.tar" + + getter, err := newGetter(endpoint, target) + require.NoError(t, err, "newGetter should not return an error") + require.NotNil(t, getter.client, "getter client should not be nil") + require.NotNil(t, getter.request, "getter request should not be nil") + require.Equal(t, endpoint, getter.request.URL().String(), "getter request URL should match the input endpoint") +} + +func TestNewDownloader(t *testing.T) { + logLevel := logging.Info + + downloader, err := NewDownloader(network.Network{ID: constants.FujiID}, logLevel) + require.NoError(t, err, "NewDownloader should not return an error") + require.NotNil(t, downloader.logger, "downloader logger should not be nil") + require.NotNil(t, downloader.getter.client, "downloader getter client should not be nil") +} + +func TestDownloader_Download(t *testing.T) { + // Mock server to simulate file download + mockData := []byte("mock file content") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(mockData) + })) + defer server.Close() + + // Create a temporary file for download target + tmpFile, err := os.CreateTemp("", "test-download-*") + require.NoError(t, err, "Temporary file creation failed") + defer os.Remove(tmpFile.Name()) + + // Ensure newGetter initializes properly + getter, err := newGetter(server.URL, tmpFile.Name()) + require.NoError(t, err, "Getter initialization failed") + + // Ensure the getter has a valid request + require.NotNil(t, getter.request, "Getter request is nil") + + // Initialize a no-op logger to avoid output + logger := logging.NoLog{} + downloader := Downloader{ + getter: getter, + logger: logger, + currentOp: &sync.Mutex{}, + } + + // Test the Download functionality + err = downloader.Download() + require.NoError(t, err, "Download should not return an error") + + // Validate the downloaded content + content, err := os.ReadFile(tmpFile.Name()) + require.NoError(t, err, "Reading downloaded file should not return an error") + require.Equal(t, mockData, content, "Downloaded file content should match the mock data") +} + +func TestDownloader_UnpackTo(t *testing.T) { + // Create a mock tar file + var buf bytes.Buffer + tarWriter := tar.NewWriter(&buf) + + files := []struct { + Name, Body string + }{ + {"file1.txt", "This is file1"}, + {"dir/file2.txt", "This is file2"}, + } + for _, file := range files { + header := &tar.Header{ + Name: file.Name, + Size: int64(len(file.Body)), + Mode: 0o600, + } + require.NoError(t, tarWriter.WriteHeader(header)) + _, err := tarWriter.Write([]byte(file.Body)) + require.NoError(t, err) + } + require.NoError(t, tarWriter.Close()) + + // Write tar file to a temporary file + tmpTar, err := os.CreateTemp("", "test-tar-*") + require.NoError(t, err) + defer os.Remove(tmpTar.Name()) + _, err = tmpTar.Write(buf.Bytes()) + require.NoError(t, err) + require.NoError(t, tmpTar.Close()) + + targetDir := t.TempDir() + + logger := logging.NoLog{} + downloader := Downloader{ + getter: Getter{ + request: &grab.Request{ + Filename: tmpTar.Name(), + }, + }, + logger: logger, + currentOp: &sync.Mutex{}, + } + + err = downloader.UnpackTo(targetDir) + require.NoError(t, err, "UnpackTo should not return an error") + + // Verify unpacked files + for _, file := range files { + filePath := filepath.Join(targetDir, file.Name) + content, err := os.ReadFile(filePath) + require.NoError(t, err, fmt.Sprintf("Reading file %s should not return an error", file.Name)) + require.Equal(t, file.Body, string(content), fmt.Sprintf("File content for %s should match", file.Name)) + } +} + +func TestDownloader_EndToEnd(t *testing.T) { + // Set up a temporary directory for testing + tmpDir := t.TempDir() + targetDir := filepath.Join(tmpDir, "extracted_files") + + // Configure the test network (Fuji in this case) + net := network.Network{ID: constants.FujiID} + + // Initialize a logger + logLevel := logging.Debug + + // Step 1: Create the downloader + downloader, err := NewDownloader(net, logLevel) + require.NoError(t, err, "Failed to initialize downloader") + + // Step 2: Start the download + t.Log("Starting download...") + err = downloader.Download() + require.NoError(t, err, "Download failed") + + // Step 3: Unpack the downloaded archive + t.Log("Unpacking downloaded archive...") + err = downloader.UnpackTo(targetDir) + require.NoError(t, err, "Failed to unpack archive") + + // Step 4: Validate the extracted files + t.Log("Validating extracted files...") + fileInfo, err := os.Stat(targetDir) + require.NoError(t, err, "Extracted directory does not exist") + require.True(t, fileInfo.IsDir(), "Extracted path is not a directory") + + // Check that at least one file is extracted + extractedFiles, err := os.ReadDir(targetDir) + require.NoError(t, err, "Failed to read extracted directory contents") + require.NotEmpty(t, extractedFiles, "No files extracted from archive") + + // Step 5: Clean up (optional since TempDir handles this automatically) + t.Log("Test completed successfully!") +} diff --git a/sdk/publicarchive/helpers.go b/sdk/publicarchive/helpers.go new file mode 100644 index 000000000..2f46029a9 --- /dev/null +++ b/sdk/publicarchive/helpers.go @@ -0,0 +1,51 @@ +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package publicarchive + +import ( + "fmt" + "os" +) + +// IsEmpty returns true if the Downloader is empty and not initialized +func (d Downloader) IsEmpty() bool { + return d.getter.client == nil +} + +func (d Downloader) GetDownloadedFilePath() (string, error) { + if d.GetBytesComplete() != d.GetDownloadSize() { + return "", fmt.Errorf("download is not completed") + } + return d.getter.request.Filename, nil +} + +// GetDownloadSize returns the size of the download +func (d Downloader) GetDownloadSize() int64 { + d.getter.mutex.RLock() + defer d.getter.mutex.RUnlock() + return d.getter.size +} + +func (d Downloader) setDownloadSize(size int64) { + d.getter.mutex.Lock() + defer d.getter.mutex.Unlock() + d.getter.size = size +} + +// GetCurrentProgress returns the current download progress +func (d Downloader) GetBytesComplete() int64 { + d.getter.mutex.RLock() + defer d.getter.mutex.RUnlock() + return d.getter.bytesComplete +} + +func (d Downloader) setBytesComplete(progress int64) { + d.getter.mutex.Lock() + defer d.getter.mutex.Unlock() + d.getter.bytesComplete = progress +} + +func (d Downloader) CleanUp() { + _ = os.Remove(d.getter.request.Filename) +}