public archive sdk

pkg/node/helper.go

@@ -7,6 +7,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"regexp"
 	"sort"
 	"strconv"
@@ -24,8 +26,11 @@ import (
 	"github.com/ava-labs/avalanche-cli/pkg/utils"
 	"github.com/ava-labs/avalanche-cli/pkg/ux"
 	"github.com/ava-labs/avalanche-cli/pkg/vm"
+	"github.com/ava-labs/avalanche-cli/sdk/network"
+	"github.com/ava-labs/avalanche-cli/sdk/publicarchive"
 	"github.com/ava-labs/avalanche-network-runner/rpcpb"
 	"github.com/ava-labs/avalanchego/api/info"
+	"github.com/ava-labs/avalanchego/utils/logging"
 	"github.com/ava-labs/avalanchego/utils/set"
 )
 
@@ -554,3 +559,75 @@ func GetNodeData(endpoint string) (
 		"0x" + hex.EncodeToString(proofOfPossession.ProofOfPossession[:]),
 		nil
 }
+
+func SeedClusterData(
+	clusterNetwork models.Network,
+	rootDir string,
+	nodeNames []string,
+) error {
+	// only fuji is supported for now
+	if clusterNetwork.Kind != models.Fuji {
+		return fmt.Errorf("unsupported network: %s", clusterNetwork.Name())
+	}
+	network := network.FujiNetwork()
+	ux.Logger.Info("downloading public archive for network %s", clusterNetwork.Name())
+	publicArcDownloader, err := publicarchive.NewDownloader(network, logging.Off) // off as we run inside of the spinner
+	if err != nil {
+		return fmt.Errorf("failed to create public archive downloader for network %s: %w", clusterNetwork.Name(), err)
+	}
+
+	if err := publicArcDownloader.Download(); err != nil {
+		return fmt.Errorf("failed to download public archive: %w", err)
+	}
+	// defer publicArcDownloader.CleanUp()
+	if path, err := publicArcDownloader.GetDownloadedFilePath(); err != nil {
+		return fmt.Errorf("failed to get downloaded file path: %w", err)
+	} else {
+		ux.Logger.Info("public archive downloaded to %s", path)
+	}
+
+	wg := sync.WaitGroup{}
+	mu := sync.Mutex{}
+	var firstErr error
+
+	for _, nodeName := range nodeNames {
+		target := filepath.Join(rootDir, nodeName, "db")
+		ux.Logger.Info("unpacking public archive to %s", target)
+
+		// Skip if target already exists
+		if _, err := os.Stat(target); err == nil {
+			ux.Logger.Info("data folder already exists at %s. Skipping...", target)
+			continue
+		}
+		wg.Add(1)
+		go func(target string) {
+			defer wg.Done()
+
+			if err := publicArcDownloader.UnpackTo(target); err != nil {
+				// Capture the first error encountered
+				mu.Lock()
+				if firstErr == nil {
+					firstErr = fmt.Errorf("failed to unpack public archive: %w", err)
+					_ = CleanUpClusterNodeData(rootDir, nodeNames)
+				}
+				mu.Unlock()
+			}
+		}(target)
+	}
+	wg.Wait()
+
+	if firstErr != nil {
+		return firstErr
+	}
+	ux.Logger.PrintToUser("Public archive unpacked to: %s", rootDir)
+	return nil
+}
+
+func CleanUpClusterNodeData(rootDir string, nodesNames []string) error {
+	for _, nodeName := range nodesNames {
+		if err := os.RemoveAll(filepath.Join(rootDir, nodeName)); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/node/local.go

@@ -335,6 +335,13 @@ func StartLocalNode(
 			}
 		}
 		if network.Kind == models.Fuji {
+			// disable indexing for fuji
+			nodeConfig[config.IndexEnabledKey] = false
+			nodeConfigBytes, err := json.Marshal(nodeConfig)
+			if err != nil {
+				return err
+			}
+			nodeConfigStr = string(nodeConfigBytes)
 			ux.Logger.PrintToUser(logging.Yellow.Wrap("Warning: Fuji Bootstrapping can take several minutes"))
 		}
 		if err := preLocalChecks(anrSettings, avaGoVersionSetting, useEtnaDevnet, globalNetworkFlags); err != nil {
@@ -416,6 +423,14 @@ func StartLocalNode(
 		ux.Logger.PrintToUser("Starting local avalanchego node using root: %s ...", rootDir)
 		spinSession := ux.NewUserSpinner()
 		spinner := spinSession.SpinToUser("Booting Network. Wait until healthy...")
+		// preseed nodes data from public archive. ignore errors
+		nodeNames := []string{}
+		for i := 1; i <= int(numNodes); i++ {
+			nodeNames = append(nodeNames, fmt.Sprintf("node%d", i))
+		}
+		err := SeedClusterData(network, rootDir, nodeNames)
+		ux.Logger.Info("seeding public archive data finished with error: %v. Ignored if any", err)
+
 		if _, err := cli.Start(ctx, avalancheGoBinPath, anrOpts...); err != nil {
 			ux.SpinFailWithError(spinner, "", err)
 			_ = DestroyLocalNode(app, clusterName)
@@ -474,6 +489,9 @@ func UpsizeLocalNode(
 		nodeConfig = map[string]interface{}{}
 	}
 	nodeConfig[config.NetworkAllowPrivateIPsKey] = true
+	if network.Kind == models.Fuji {
+		nodeConfig[config.IndexEnabledKey] = false // disable index for Fuji
+	}
 	nodeConfigBytes, err := json.Marshal(nodeConfig)
 	if err != nil {
 		return "", err
@@ -555,6 +573,8 @@ func UpsizeLocalNode(
 
 	spinSession := ux.NewUserSpinner()
 	spinner := spinSession.SpinToUser("Creating new node with name %s on local machine", newNodeName)
+	err = SeedClusterData(network, rootDir, []string{newNodeName})
+	ux.Logger.Info("seeding public archive data finished with error: %v. Ignored if any", err)
 	// add new local node
 	if _, err := cli.AddNode(ctx, newNodeName, avalancheGoBinPath, anrOpts...); err != nil {
 		ux.SpinFailWithError(spinner, "", err)

sdk/constants/constants.go

@@ -10,5 +10,6 @@ const (
 	APIRequestLargeTimeout = 2 * time.Minute
 
 	// node
-	WriteReadUserOnlyPerms = 0o600
+	WriteReadUserOnlyPerms    = 0o600
+	WriteReadUserOnlyDirPerms = 0o700
 )

sdk/publicarchive/README.md

@@ -0,0 +1,55 @@
+# Public Archive Downloader SDK
+
+This Go package provides a utility to download and extract tar archives from public URLs. It's tailored for downloading Avalanche network archives but can be adapted for other use cases.
+
+
+## Features
+
+* Downloads files from predefined URLs.
+* Tracks download progress and logs status updates.
+* Safely unpacks .tar archives to a target directory.
+* Includes security checks to prevent path traversal and manage large files.
+
+## Usage example
+
+```
+// Copyright (C) 2024, Ava Labs, Inc. All rights reserved
+// See the file LICENSE for licensing terms.
+
+```
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ava-labs/avalanche-cli/sdk/network"
+	"github.com/ava-labs/avalanchego/utils/constants"
+	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/your-repo-name/publicarchive"
+)
+
+func main() {
+	// Initialize the downloader
+	downloader, err := publicarchive.NewDownloader(network.FujiNetwork(), logging.Debug)
+	if err != nil {
+		fmt.Printf("Failed to create downloader: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Start downloading
+	if err := downloader.Download(); err != nil {
+		fmt.Printf("Download failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Specify the target directory for unpacking
+	targetDir := "./extracted_files"
+	if err := downloader.UnpackTo(targetDir); err != nil {
+		fmt.Printf("Failed to unpack archive: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Files successfully unpacked to %s\n", targetDir)
+}
+```