Merge pull request #5194 from Algo-devops-service/relbeta3.15.0

.circleci/config.yml

@@ -18,7 +18,7 @@ parameters:
     default: "/opt/cibuild"
   result_path:
     type: string
-    default: "/tmp/build_test_results"
+    default: "/tmp/build_test_results_<< pipeline.id >>"
   valid_nightly_branch:
     type: string
     default: /hotfix\/.*/
@@ -251,7 +251,7 @@ jobs:
     parameters:
       platform:
         type: string
-    executor: << parameters.platform >>_medium
+    executor: << parameters.platform >>_large
     working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 4
     steps:
@@ -705,9 +705,6 @@ commands:
                 TestAlgohWithExpect \
                 TestGoalWithExpect \
                 TestTealdbgWithExpect
-      - store_artifacts:
-          path: << parameters.result_path >>/<< parameters.result_subdir >>
-          destination: << parameters.result_subdir >>/combined-test-results
 
   upload_binaries_command:
     description: save build artifacts for potential deployments

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,28 @@
+---
+name: Enhancement
+about: How can we enhance existent functionality, constructs, processes etc.
+title: ''
+labels: Enhancement
+assignees: ''
+
+---
+
+## Status
+
+<!-- What is the current status? -->
+
+## Expected
+
+<!-- What is expected? -->
+
+## Solution
+
+<!-- Do you have a potential/suggested solution? Document more than one if possible. -->
+
+## Dependencies
+
+<!-- Does the solution have any team or design dependencies? -->
+
+## Urgency
+
+<!-- What is the urgency here and why? -->

.github/.release.yml → .github/release.yml

@@ -18,3 +18,4 @@ changelog:
     - title: Other
       labels:
         - "*"
+

.github/workflows/container.yml

@@ -5,13 +5,11 @@ on:
   push:
     branches:
       - master
-      - rel/*
       - feature/*
-    tags:
-      - "*"
 
 jobs:
-  build:
+  build-and-push:
+    name: Build and Push to DockerHub
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
@@ -25,9 +23,7 @@ jobs:
             docker.io/${{ github.repository_owner }}/algod
           tags: |
             type=sha,format=long,prefix=
-            type=ref,event=tag
             type=ref,event=branch
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'rel/stable') }}
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -55,3 +51,20 @@ jobs:
             SHA=${{ github.sha }}
             URL=${{ github.server_url }}/${{ github.repository }}.git
             BRANCH=${{ github.ref_name }}
+
+  # TODO: uncomment when https://github.com/docker/hub-tool/issues/172 is complete
+  # update-repo-description:
+  #   name: Update DockerHub Repository Description
+  #   runs-on: ubuntu-latest
+  #   if: github.ref == format('refs/heads/{0}', 'master')
+  #   steps:
+  #     - name: Checkout Code
+  #       uses: actions/checkout@v3
+
+  #     - name: Update DockerHub Repository Description
+  #       uses: peter-evans/dockerhub-description@v3
+  #       with:
+  #         username: ${{ secrets.DOCKERHUB_USERNAME }}
+  #         password: ${{ secrets.DOCKERHUB_TOKEN }}
+  #         repository: ${{ github.repository_owner }}/algod
+  #         readme-filepath: ./docker/README.md

.gitignore

@@ -48,6 +48,7 @@ crypto/libs
 *~
 *.swp
 *.swo
+*.swn
 
 # Mac
 .DS_Store

.golangci-warnings.yml

@@ -6,12 +6,15 @@ linters:
   disable-all: true
   enable:
     - deadcode
+    - gosec
     - partitiontest
     - structcheck
     - varcheck
     - unused
 
 linters-settings:
+  gosec: # we are mostly only interested in G601
+    excludes: [G101, G103, G104, G107, G202, G301, G302, G303, G304, G306, G307, G404]
   custom:
     partitiontest:
       path: cmd/partitiontest_linter/plugin.so
@@ -51,6 +54,7 @@ issues:
     - path: _test\.go
       linters:
         - deadcode
+        - gosec
         - structcheck
         - varcheck
         - unused

.golangci.yml

@@ -7,6 +7,7 @@ linters:
   disable-all: true
   enable:
     - errcheck
+    - exportloopref
     - gofmt
     - gosimple
     - govet
@@ -107,6 +108,7 @@ issues:
     - path: _test\.go
       linters:
         - errcheck
+        # - exportloopref
         # - gofmt
         - gosimple
         # - govet
@@ -117,61 +119,55 @@ issues:
         - staticcheck
         - typecheck
     # Ignore missing parallel tests in existing packages
-    - path: agreement.*_test\.go
+    - path: ^agreement.*_test\.go
       linters:
         - paralleltest
-    - path: catchup.*_test\.go
+    - path: ^catchup.*_test\.go
       linters:
         - paralleltest
-    - path: cmd.*_test\.go
+    - path: ^config.*_test\.go
       linters:
         - paralleltest
-    - path: config.*_test\.go
+    - path: ^crypto.*_test\.go
       linters:
         - paralleltest
-    - path: crypto.*_test\.go
+    - path: ^daemon.*_test\.go
       linters:
         - paralleltest
-    - path: daemon.*_test\.go
+    - path: ^data.*_test\.go
       linters:
         - paralleltest
-    - path: data.*_test\.go
+    - path: ^gen.*_test\.go
       linters:
         - paralleltest
-    - path: gen.*_test\.go
+    - path: ^ledger.*_test\.go
       linters:
         - paralleltest
-    - path: ledger.*_test\.go
+    - path: ^logging.*_test\.go
       linters:
         - paralleltest
-    - path: logging.*_test\.go
+    - path: ^network.*_test\.go
       linters:
         - paralleltest
-    - path: netdeploy.*_test\.go
+    - path: ^node.*_test\.go
       linters:
         - paralleltest
-    - path: network.*_test\.go
+    - path: ^protocol.*_test\.go
       linters:
         - paralleltest
-    - path: node.*_test\.go
+    - path: ^rpcs.*_test\.go
       linters:
         - paralleltest
-    - path: protocol.*_test\.go
+    - path: ^stateproof.*_test\.go
       linters:
         - paralleltest
-    - path: rpcs.*_test\.go
+    - path: ^test.*_test\.go
       linters:
         - paralleltest
-    - path: stateproof.*_test\.go
+    - path: ^tools.*_test\.go
       linters:
         - paralleltest
-    - path: test.*_test\.go
-      linters:
-        - paralleltest
-    - path: tools.*_test\.go
-      linters:
-        - paralleltest
-    - path: util.*_test\.go
+    - path: ^util.*_test\.go
       linters:
         - paralleltest
     # Add all linters here -- Comment this block out for testing linters

CONTRIBUTING.md

@@ -15,7 +15,7 @@ Some of our most active projects include:
 * [py-algorand-sdk](https://github.com/algorand/py-algorand-sdk) - Python SDK
 * [sandbox](https://github.com/algorand/sandbox) - Algorand node quickstart tool
 
-# Filing Issues
+## Filing Issues
 
 Did you discover a bug? Do you have a feature request? Filing issues is an easy way anyone can contribute and helps us improve Algorand. We use GitHub Issues to track all known bugs and feature requests.
 
@@ -25,23 +25,21 @@ If you’d like to contribute to any of the repositories, please file a [GitHub
 
 See the GitHub help guide for more information on [filing an issue](https://help.github.com/en/articles/creating-an-issue).
 
-## Vulnerabilities
+## Security / Vulnerabilities
 
-Please don't create issues for any security vulnerabilities.  Instead, we would appreciate it if you reported them through our [vulnerability disclosure form][vuln_url].  This allows us to distribute a fix before the vulnerability is exploited.
-
-Additionally, if you believe that you've discovered a security vulnerability, you might qualify for our bug bounty program.  Visit our [bug bounty site][bug_bounty_url] for details.
+Please refer to our [SECURITY](SECURITY.md) document.
 
 If you have any questions, don't hesitate to contact us at [email protected].
 
-# Contribution Model
+## Contribution Model
 
 For each of our repositories we use the same model for contributing code. Developers wanting to contribute must create pull requests. This process is described in the GitHub [Creating a pull request from a fork](https://help.github.com/en/articles/creating-a-pull-request-from-a-fork) documentation. Each pull request should be initiated against the `master` branch in the Algorand repository.  After a pull request is submitted the core development team will review the submission and communicate with the developer using the comments sections of the PR. After the submission is reviewed and approved, it will be merged into the `master` branch of the source. These changes will be merged to our release branch on the next viable release date. For the SDKs, this may be immediate. Changes to the node software may take more time as we must ensure and verify the security, as well as apply protocol upgrades in an orderly way.
 
 Note: some of our projects are using gitflow, for these you will open pull requests against the `develop` branch.
 
 Again, if you have a patch for a critical security vulnerability, please use our [vulnerability disclosure form][vuln_url] instead of creating a PR.  We'll follow up with you on distributing the patch before we merge it.
 
-# Code Guidelines
+## Code Guidelines
 
 For Go code we use the [Golang guidelines defined here](https://golang.org/doc/effective_go.html).
 * Code must adhere to the official Go formatting guidelines (i.e. uses gofmt).
@@ -52,7 +50,7 @@ For JavaScript code we use the [MDN formatting rules](https://developer.mozilla.
 
 For Java code we use [Oracle’s standard formatting rules for Java](https://www.oracle.com/technetwork/java/codeconventions-150003.pdf).
 
-# Communication Channels
+## Communication Channels
 
 The core development team monitors the Algorand community forums and regularly responds to questions and suggestions. Issues and Pull Requests are handled on GitHub.
 

Dockerfile

@@ -43,7 +43,7 @@ RUN /dist/files/build/install.sh \
 
 FROM debian:bullseye-slim as final
 
-ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" ALGORAND_DATA="/algod/data"
+ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" KMD_PORT="7833" ALGORAND_DATA="/algod/data"
 
 # curl is needed to lookup the fast catchup url
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \
@@ -58,8 +58,8 @@ USER algorand
 COPY --chown=algorand:algorand --from=builder "/dist/bin/" "/node/bin/"
 COPY --chown=algorand:algorand --from=builder "/dist/files/run/" "/node/run/"
 
-# Expose Algod REST API, Algod Gossip, and Prometheus Metrics ports
-EXPOSE $ALGOD_PORT 4160 9100
+# Expose Algod REST API, KMD REST API, Algod Gossip, and Prometheus Metrics ports
+EXPOSE $ALGOD_PORT $KMD_PORT 4160 9100
 
 WORKDIR /algod
 

Makefile

@@ -85,7 +85,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./ledger/store ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
+MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
 
 default: build
 

agreement/demux.go

@@ -201,7 +201,7 @@ func (d *demux) next(s *Service, deadline time.Duration, fastDeadline time.Durat
 		switch e.t() {
 		case payloadVerified:
 			e = e.(messageEvent).AttachValidatedAt(s.Clock.Since())
-		case payloadPresent:
+		case payloadPresent, votePresent:
 			e = e.(messageEvent).AttachReceivedAt(s.Clock.Since())
 		}
 	}()

agreement/events.go

@@ -146,7 +146,7 @@ const (
 	// that a certificate has formed for that proposal-value.
 	proposalCommittable
 
-	// proposalCommittable is returned by the proposal state machines when a
+	// proposalAccepted is returned by the proposal state machines when a
 	// proposal-value is accepted.
 	proposalAccepted
 
@@ -946,7 +946,22 @@ func (e messageEvent) AttachValidatedAt(d time.Duration) messageEvent {
 	return e
 }
 
+// AttachReceivedAt looks for an unauthenticatedProposal inside a
+// payloadPresent or votePresent messageEvent, and attaches the given
+// time to the proposal's receivedAt field.
 func (e messageEvent) AttachReceivedAt(d time.Duration) messageEvent {
-	e.Input.UnauthenticatedProposal.receivedAt = d
+	if e.T == payloadPresent {
+		e.Input.UnauthenticatedProposal.receivedAt = d
+	} else if e.T == votePresent {
+		// Check for non-nil Tail, indicating this votePresent event
+		// contains a synthetic payloadPresent event that was attached
+		// to it by setupCompoundMessage.
+		if e.Tail != nil && e.Tail.T == payloadPresent {
+			// The tail event is payloadPresent, serialized together
+			// with the proposal vote as a single CompoundMessage
+			// using a protocol.ProposalPayloadTag network message.
+			e.Tail.Input.UnauthenticatedProposal.receivedAt = d
+		}
+	}
 	return e
 }