reduce amount of data loaded during atx validation

.github/workflows/api-swagger-ui.yml

@@ -0,0 +1,72 @@
+name: Build and Push Swagger-UI to R2 testnet-api-docs.spacemesh.network
+
+env:
+  go-version: "1.22"
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  check-version:
+    runs-on: ubuntu-22.04
+    outputs:
+      go-sm-api-version: ${{ steps.go-sm-api-version.outputs.GO_SM_API_VERSION }}
+    steps:
+    - name: Checkout target repository on last release
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        check-latest: true
+        go-version: ${{ env.go-version }}
+
+    - name: Extract dependency version
+      id: go-sm-api-version
+      run: |
+        version=$(go list -m 'github.com/spacemeshos/api/release/go' | awk '{print $2}')
+        echo "GO_SM_API_VERSION=$version" > $GITHUB_OUTPUT
+
+  deploy:
+    runs-on: ubuntu-22.04
+    needs: check-version
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        repository: spacemeshos/api
+        path: api
+        fetch-depth: 0
+        ref: 'refs/tags/${{ needs.check-version.outputs.go-sm-api-version }}'
+
+    - name: upload to testnet
+      uses: jakejarvis/s3-sync-action@master
+      with:
+        args: --acl public-read --follow-symlinks --delete
+      env:
+          AWS_S3_BUCKET: ${{ secrets.CLOUDFLARE_TESTNET_API_BUCKET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_SECRET_ACCESS_KEY }}
+          SOURCE_DIR: api/release/openapi/swagger/src
+          DEST_DIR: '/${{ github.event.release.tag_name }}'
+          AWS_S3_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+
+    - name: update url json file for testnet
+      working-directory: api/release/openapi/swagger/src
+      run: |
+        curl -o spec_urls.json https://testnet-api-docs.spacemesh.network/spec_urls.json
+        new_url="{\"url\":\"https://testnet-api-docs.spacemesh.network/${{ github.event.release.tag_name }}/api.swagger.json\",\"name\":\"${{ github.event.release.tag_name }}\"}"
+        jq ". += [$new_url]" spec_urls.json > tmp.json && mv tmp.json spec_urls.json
+
+    - name: upload new testnet json file
+      uses: jakejarvis/s3-sync-action@master
+      with:
+        args: --acl public-read --follow-symlinks --delete
+      env:
+          AWS_S3_BUCKET: ${{ secrets.CLOUDFLARE_TESTNET_API_BUCKET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_SECRET_ACCESS_KEY }}
+          SOURCE_DIR: api/release/openapi/swagger/src/spec_urls.json
+          DEST_DIR: ''
+          AWS_S3_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+

CHANGELOG.md

@@ -2,6 +2,97 @@
 
 See [RELEASE](./RELEASE.md) for workflow instructions.
 
+## Release v1.6.8
+
+### Improvements
+
+* [#6269](https://github.com/spacemeshos/go-spacemesh/pull/6269) Use sub query instead of left join to speed up
+  transaction list query.
+
+* [#6278](https://github.com/spacemeshos/go-spacemesh/pull/6278) Reduce logging levels of some messages to reduce noise.
+
+* [#6279](https://github.com/spacemeshos/go-spacemesh/pull/6279) Speed up initialization of tortoise by reducing the
+  number of queries to the DB.
+
+* [#6268](https://github.com/spacemeshos/go-spacemesh/pull/6268) Fix incorrect timing collection for proposal builder.
+
+* [#6278](https://github.com/spacemeshos/go-spacemesh/pull/6278) Reduce logging levels of some messages to reduce noise.
+
+## Release v1.6.7
+
+### Improvements
+
+* [#6197](https://github.com/spacemeshos/go-spacemesh/pull/6197) Fix falling back to poet PoW if recertification
+  failed after getting a 401 on registering
+
+* [#6241](https://github.com/spacemeshos/go-spacemesh/pull/6241) Improve speed of ATX cache warmup.
+
+### Features
+
+* [#6213](https://github.com/spacemeshos/go-spacemesh/pull/6213) Adds `labels_per_unit` to the v2alpha1 Network Info API.
+
+## Release v1.6.6-hotfix1
+
+### Improvements
+
+* [#6248](https://github.com/spacemeshos/go-spacemesh/pull/6248) Fixed node not being able to handle more than 6.55M
+  ATXs per epoch.
+
+## Release v1.6.6
+
+### Improvements
+
+* [#6198](https://github.com/spacemeshos/go-spacemesh/pull/6198) Configure default TTL for caching poet's /v1/info
+
+* [#6199](https://github.com/spacemeshos/go-spacemesh/pull/6199) Cache poet's /v1/pow_params
+
+## Release v1.6.5
+
+### Improvements
+
+* [#6185](https://github.com/spacemeshos/go-spacemesh/pull/6185) Optimize mempool
+
+* [#6187](https://github.com/spacemeshos/go-spacemesh/pull/6187) The merge tool now ignores files that are not `.key`
+  files in the `identities` directory when merging two nodes.
+
+* [#6128](https://github.com/spacemeshos/go-spacemesh/pull/6128) Reduce logs spam
+
+## Release v1.6.4
+
+### Improvements
+
+* [#6107](https://github.com/spacemeshos/go-spacemesh/pull/6107) Cache PoET queries between multiple identities on the
+  same node. This will reduce the number of requests the node makes to the PoET server during the cyclegap.
+
+* [#6152](https://github.com/spacemeshos/go-spacemesh/pull/6152) Fixed a bug where in rare cases the node would panic
+  due to the closing of a closed channel in the fetcher.
+
+* [#6142](https://github.com/spacemeshos/go-spacemesh/pull/6142) Fix node not dropping peers that are broadcasting
+  invalid ATXs.
+
+## Release v1.6.3
+
+### Improvements
+
+* [#6137](https://github.com/spacemeshos/go-spacemesh/pull/6137) Fix hanging ATX sync.
+
+## Release v1.6.2
+
+### Improvements
+
+* [#5793](https://github.com/spacemeshos/go-spacemesh/pull/5793) Reduced hare committee 8x from 400 to 50 to decrease
+  network traffic caused by Hare.
+
+* [#6099](https://github.com/spacemeshos/go-spacemesh/pull/6099) Adds new metrics to the API to provide insights into
+  the performance and behavior of the node's APIs.
+
+* [#6115](https://github.com/spacemeshos/go-spacemesh/pull/6115) Increase the number of supported ATXs to 8.0 Mio.
+
+### Features
+
+* [#6112](https://github.com/spacemeshos/go-spacemesh/pull/6112) Adds vesting, vault, and drain vault contents to the
+  v2alpha2 Transaction API. Fixes the 'unspecified' transaction type.
+
 ## Release v1.6.1
 
 ### Improvements

activation/activation.go

@@ -45,13 +45,17 @@ type PoetConfig struct {
 	RequestTimeout                 time.Duration `mapstructure:"poet-request-timeout"`
 	RequestRetryDelay              time.Duration `mapstructure:"retry-delay"`
 	PositioningATXSelectionTimeout time.Duration `mapstructure:"positioning-atx-selection-timeout"`
+	CertifierInfoCacheTTL          time.Duration `mapstructure:"certifier-info-cache-ttl"`
+	PowParamsCacheTTL              time.Duration `mapstructure:"pow-params-cache-ttl"`
 	MaxRequestRetries              int           `mapstructure:"retry-max"`
 }
 
 func DefaultPoetConfig() PoetConfig {
 	return PoetConfig{
-		RequestRetryDelay: 400 * time.Millisecond,
-		MaxRequestRetries: 10,
+		RequestRetryDelay:     400 * time.Millisecond,
+		MaxRequestRetries:     10,
+		CertifierInfoCacheTTL: 5 * time.Minute,
+		PowParamsCacheTTL:     5 * time.Minute,
 	}
 }
 
@@ -415,7 +419,13 @@ func (b *Builder) run(ctx context.Context, sig *signing.EdSigner) {
 	for _, poet := range b.poets {
 		eg.Go(func() error {
 			_, err := poet.Certify(ctx, sig.NodeID())
-			if err != nil {
+			switch {
+			case errors.Is(err, ErrCertificatesNotSupported):
+				b.logger.Debug("not certifying (not supported in poet)",
+					log.ZShortStringer("smesherID", sig.NodeID()),
+					zap.String("poet", poet.Address()),
+				)
+			case err != nil:
 				b.logger.Warn("failed to certify poet", zap.Error(err), log.ZShortStringer("smesherID", sig.NodeID()))
 			}
 			return nil
@@ -500,7 +510,6 @@ func (b *Builder) BuildNIPostChallenge(ctx context.Context, nodeID types.NodeID)
 		// build new challenge
 		logger.Info("building new NiPOST challenge", zap.Uint32("current_epoch", current.Uint32()))
 	case err != nil:
-		logger.Info("failed to load NiPoST challenge from local state", zap.Error(err))
 		return nil, fmt.Errorf("get nipost challenge: %w", err)
 	case challenge.PublishEpoch < current:
 		logger.Info(
@@ -519,7 +528,7 @@ func (b *Builder) BuildNIPostChallenge(ctx context.Context, nodeID types.NodeID)
 		}
 	default:
 		// challenge is fresh
-		logger.Info("loaded NiPoST challenge from local state",
+		logger.Debug("loaded NiPoST challenge from local state",
 			zap.Uint32("current_epoch", current.Uint32()),
 			zap.Uint32("publish_epoch", challenge.PublishEpoch.Uint32()),
 		)
@@ -624,7 +633,7 @@ func (b *Builder) BuildNIPostChallenge(ctx context.Context, nodeID types.NodeID)
 			PositioningATX: posAtx,
 		}
 	}
-	logger.Info("persisting the new NiPOST challenge", zap.Object("challenge", challenge))
+	logger.Debug("persisting the new NiPOST challenge", zap.Object("challenge", challenge))
 	if err := nipost.AddChallenge(b.localDB, nodeID, challenge); err != nil {
 		return nil, fmt.Errorf("add nipost challenge: %w", err)
 	}
@@ -685,7 +694,6 @@ func (b *Builder) PublishActivationTx(ctx context.Context, sig *signing.EdSigner
 		return fmt.Errorf("wait for publication epoch: %w", ctx.Err())
 	case <-b.layerClock.AwaitLayer(challenge.PublishEpoch.FirstLayer()):
 	}
-	b.logger.Debug("publication epoch has arrived!", log.ZShortStringer("smesherID", sig.NodeID()))
 
 	for {
 		b.logger.Info(
@@ -915,13 +923,13 @@ func (b *Builder) getPositioningAtx(
 		return types.EmptyATXID, err
 	}
 
-	b.logger.Info("found candidate positioning atx",
+	b.logger.Debug("found candidate positioning atx",
 		log.ZShortStringer("id", id),
 		log.ZShortStringer("smesherID", nodeID),
 	)
 
 	if previous == nil {
-		b.logger.Info("selected atx as positioning atx",
+		b.logger.Info("selected positioning atx",
 			log.ZShortStringer("id", id),
 			log.ZShortStringer("smesherID", nodeID))
 		return id, nil
@@ -999,14 +1007,14 @@ func findFullyValidHighTickAtx(
 
 	// iterate trough epochs, to get first valid, not malicious ATX with the biggest height
 	atxdata.IterateHighTicksInEpoch(publish+1, func(id types.ATXID) (contSearch bool) {
-		logger.Info("found candidate for high-tick atx", log.ZShortStringer("id", id))
+		logger.Debug("found candidate for high-tick atx", log.ZShortStringer("id", id))
 		if ctx.Err() != nil {
 			return false
 		}
 		// verify ATX-candidate by getting their dependencies (previous Atx, positioning ATX etc.)
 		// and verifying PoST for every dependency
 		if err := validator.VerifyChain(ctx, id, goldenATXID, opts...); err != nil {
-			logger.Info("rejecting candidate for high-tick atx", zap.Error(err), log.ZShortStringer("id", id))
+			logger.Debug("rejecting candidate for high-tick atx", zap.Error(err), log.ZShortStringer("id", id))
 			return true
 		}
 		found = &id

activation/certifier.go

@@ -117,7 +117,15 @@ func (c *Certifier) Certificate(
 		case !errors.Is(err, sql.ErrNotFound):
 			return nil, fmt.Errorf("getting certificate from DB for: %w", err)
 		}
-		return c.Recertify(ctx, id, certifier, pubkey)
+		cert, err = c.client.Certify(ctx, id, certifier, pubkey)
+		if err != nil {
+			return nil, fmt.Errorf("certifying POST at %v: %w", certifier, err)
+		}
+
+		if err := certifierdb.AddCertificate(c.db, id, *cert, pubkey); err != nil {
+			c.logger.Warn("failed to persist poet cert", zap.Error(err))
+		}
+		return cert, nil
 	})
 
 	if err != nil {
@@ -126,21 +134,11 @@ func (c *Certifier) Certificate(
 	return cert.(*certifierdb.PoetCert), nil
 }
 
-func (c *Certifier) Recertify(
-	ctx context.Context,
-	id types.NodeID,
-	certifier *url.URL,
-	pubkey []byte,
-) (*certifierdb.PoetCert, error) {
-	cert, err := c.client.Certify(ctx, id, certifier, pubkey)
-	if err != nil {
-		return nil, fmt.Errorf("certifying POST at %v: %w", certifier, err)
-	}
-
-	if err := certifierdb.AddCertificate(c.db, id, *cert, pubkey); err != nil {
-		c.logger.Warn("failed to persist poet cert", zap.Error(err))
+func (c *Certifier) DeleteCertificate(id types.NodeID, pubkey []byte) error {
+	if err := certifierdb.DeleteCertificate(c.db, id, pubkey); err != nil {
+		return err
 	}
-	return cert, nil
+	return nil
 }
 
 type CertifierClient struct {
@@ -209,7 +207,7 @@ func (c *CertifierClient) obtainPostFromLastAtx(ctx context.Context, nodeId type
 		}
 	}
 
-	c.logger.Info("found POST in an existing ATX", zap.String("atx_id", atxid.Hash32().ShortString()))
+	c.logger.Debug("found POST in an existing ATX", zap.String("atx_id", atxid.Hash32().ShortString()))
 	return &nipost.Post{
 		Nonce:         post.Nonce,
 		Indices:       post.Indices,
@@ -222,21 +220,21 @@ func (c *CertifierClient) obtainPostFromLastAtx(ctx context.Context, nodeId type
 }
 
 func (c *CertifierClient) obtainPost(ctx context.Context, id types.NodeID) (*nipost.Post, error) {
-	c.logger.Info("looking for POST for poet certification")
+	c.logger.Debug("looking for POST for poet certification")
 	post, err := nipost.GetPost(c.localDb, id)
 	switch {
 	case err == nil:
-		c.logger.Info("found POST in local DB")
+		c.logger.Debug("found POST in local DB")
 		return post, nil
 	case errors.Is(err, sql.ErrNotFound):
 		// no post found
 	default:
 		return nil, fmt.Errorf("loading initial post from db: %w", err)
 	}
 
-	c.logger.Info("POST not found in local DB. Trying to obtain POST from an existing ATX")
+	c.logger.Debug("POST not found in local DB. Trying to obtain POST from an existing ATX")
 	if post, err := c.obtainPostFromLastAtx(ctx, id); err == nil {
-		c.logger.Info("found POST in an existing ATX")
+		c.logger.Debug("found POST in an existing ATX")
 		if err := nipost.AddPost(c.localDb, id, *post); err != nil {
 			c.logger.Error("failed to save post", zap.Error(err))
 		}

activation/e2e/checkpoint_test.go

@@ -188,7 +188,7 @@ func TestCheckpoint_PublishingSoloATXs(t *testing.T) {
 	// 3. Spawn new ATX handler and builder using the new DB
 	poetDb = activation.NewPoetDb(newDB, logger.Named("poetDb"))
 	cdb = datastore.NewCachedDB(newDB, logger)
-	atxdata, err = atxsdata.Warm(newDB, 1)
+	atxdata, err = atxsdata.Warm(newDB, 1, logger)
 	poetService = activation.NewPoetServiceWithClient(poetDb, client, poetCfg, logger)
 	validator = activation.NewValidator(newDB, poetDb, cfg, opts.Scrypt, verifier)
 	require.NoError(t, err)

activation/e2e/poet_test.go

@@ -40,7 +40,7 @@ func (h *HTTPPoetTestHarness) Client(
 	db *activation.PoetDb,
 	cfg activation.PoetConfig,
 	logger *zap.Logger,
-	opts ...activation.PoetClientOpt,
+	opts ...activation.PoetServiceOpt,
 ) (activation.PoetService, error) {
 	return activation.NewPoetService(
 		db,