Merge pull request #1 from foundryservices/wait

Wait

.gitignore

@@ -2,4 +2,5 @@
 *.cdc
 *.checkpoint
 *.log
-flow-go/
+flow-go/
+.vscode/

README.md

@@ -1 +1,99 @@
 # flow-rosetta
+forked from [dapperlabs/flow-rosetta](https://github.com/dapperlabs/flow-rosetta)
+
+## Description
+
+The Flow Rosetta Server implements the Rosetta Data API specifications for the Flow network.
+It uses the Flow DPS Server's GRPC API as the backend to query the required data.
+Flow core contract addresses are derived from the chain ID with which the service is started.
+This allows the Rosetta API to access state remotely, or locally by running the Flow DPS Server on the same host.
+
+## Dependencies
+
+Go `v1.16` or higher is required to compile `flow-dps-rosetta`.
+Only Linux amd64 builds are supported, because of the dependency to the [`flow-go/crypto`](https://github.com/onflow/flow-go/tree/master/crypto) package.
+
+In order to build the live binary, the following extra steps and dependencies are required:
+
+* [`CMake`](https://cmake.org/install/)
+
+Please note that the flow-go repository should be cloned into this projects directory with its default name, so that the Go module replace statement works as intended: `replace github.com/onflow/flow-go/crypto => ./flow-go/crypto`.
+
+* `git clone [email protected]:onflow/flow-go.git`
+* `cd flow-go/crypto`
+* `git checkout v0.21.4`
+* `go generate`
+
+You can then verify that the installation of the flow-go crypto package has been successful by running the tests of the project.
+
+## Testing
+
+The Flow system model, where resources can be moved freely between accounts without generating events, makes it impossible to fully reconcile account balances on the Rosetta Data API.
+As a consequence, balance reconciliation has to be disabled when running the Rosetta CLI against the Flow Rosetta Server.
+
+Additionally, some of the events generated by Flow are not accurately reflecting the account address they relate to.
+This is also due to the same resource-based smart contract programming model.
+For instance, when an account receives vaults from different locations to execute a swap of tokens, the events related to this swap might indicate the swap
+contract's address, as it uses volatile vaults.
+
+Currently, the only way to work around this issue is to create exemptions for accounts which contain such smart contracts.
+As account balances using non-standard approaches to transfer Flow tokens can already not be reconciled, this is an acceptable limitation.
+In general, transactions by Rosetta should not be used to deduce account balances.
+Full historical account balance lookup is available and should thus be prefered to determine the account balance at any block height.
+
+The discussed configuration is available in the `flow.json` and `exemptions.json` files for the `mainnet-9` spork DPS.
+The following command can be executed to validate the Data API for that spork:
+
+```sh
+rosetta-cli check:data --configuration-file flow.json
+```
+
+## Usage
+
+```sh
+Usage of flow-rosetta-server:
+  -a, --api string              host URL for GRPC API endpoint (default "127.0.0.1:5005")
+  -e, --cache uint              maximum cache size for register reads in bytes (default 1073741824)
+  -l, --level string            log output level (default "info")
+  -p, --port uint16             port to host Rosetta API on (default 8080)
+  -t, --transaction-limit int   maximum amount of transactions to include in a block response (default 200)
+      --smart-status-codes      enable smart non-500 HTTP status codes for Rosetta API errors
+```
+
+## Example
+
+The following command line starts the Flow Rosetta server for a main network spork on port 8080.
+It uses a local instance of the Flow DPS Server for access to the execution state.
+
+```sh
+./flow-rosetta-server -a "127.0.0.1:5005" -p 8080
+```
+
+## Architecture
+
+The Rosetta API needs its own documentation because of the amount of components it has that interact with each other.
+The main reason for its complexity is that it needs to interact with the Flow Virtual Machine (FVM) and to translate between the Flow and Rosetta application domains.
+
+### Invoker
+
+This component, given a Cadence script, can execute it at any given height and return the value produced by the script.
+
+[Package documentation](https://pkg.go.dev/github.com/optakt/flow-dps-rosetta/service/invoker)
+
+### Retriever
+
+The retriever uses the other components to retrieve account balances, blocks and transactions.
+
+[Package documentation](https://pkg.go.dev/github.com/optakt/flow-dps-rosetta/service/retriever)
+
+### Scripts
+
+The script package produces Cadence scripts with the correct imports and storage paths, depending on the configured Flow chain ID.
+
+[Package documentation](https://pkg.go.dev/github.com/optakt/flow-dps-rosetta/service/scripts)
+
+### Validator
+
+The Validator component validates whether the given Rosetta identifiers are valid.
+
+[Package documentation](https://pkg.go.dev/github.com/optakt/flow-dps-rosetta/service/validator)

cmd/flow-rosetta-server/flow.json

@@ -30,7 +30,7 @@
   "interesting_accounts": "",
   "reconciliation_disabled": true,
   "reconciliation_drain_disabled": false,
-  "inactive_discrepency_search_disabled": false,
+  "inactive_discrepancy_search_disabled": false,
   "balance_tracking_disabled": false,
   "coin_tracking_disabled": false,
   "status_port": 9090,

cmd/flow-rosetta-server/main.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
 	"github.com/rs/zerolog"
 	"github.com/spf13/pflag"
 	"github.com/ziflex/lecho/v2"
@@ -69,6 +70,8 @@ func run() int {
 		flagPort         uint16
 		flagTransactions uint
 		flagSmart        bool
+		flagWait         bool
+		flagDump         bool
 	)
 
 	pflag.StringVarP(&flagDPS, "dps-api", "a", "127.0.0.1:5005", "host address for GRPC API endpoint")
@@ -78,6 +81,8 @@ func run() int {
 	pflag.Uint16VarP(&flagPort, "port", "p", 8080, "port to host Rosetta API on")
 	pflag.UintVarP(&flagTransactions, "transaction-limit", "t", 200, "maximum amount of transactions to include in a block response")
 	pflag.BoolVar(&flagSmart, "smart-status-codes", false, "enable smart non-500 HTTP status codes for Rosetta API errors")
+	pflag.BoolVar(&flagDump, "dump-requests", false, "print out full request and responses")
+	pflag.BoolVarP(&flagWait, "wait-for-index", "w", false, "wait for index to be available instead of quitting right away, useful when DPS Live index bootstraps")
 
 	pflag.Parse()
 
@@ -105,10 +110,15 @@ func run() int {
 	dpsAPI := api.NewAPIClient(conn)
 	index := api.IndexFromAPI(dpsAPI, codec)
 
+wait:
 	// Deduce chain ID from DPS API to configure parameters for script exec.
 	first, err := index.First()
 	if err != nil {
 		log.Error().Err(err).Msg("could not get first height from DPS API")
+		if flagWait {
+			time.Sleep(30 * time.Second)
+			goto wait
+		}
 		return failure
 	}
 	root, err := index.Header(first)
@@ -169,7 +179,22 @@ func run() int {
 	server.HideBanner = true
 	server.HidePort = true
 	server.Logger = elog
-	server.Use(lecho.Middleware(lecho.Config{Logger: elog}))
+
+	logger := lecho.Middleware(lecho.Config{Logger: elog})
+
+	if flagDump {
+		logger = middleware.BodyDump(func(c echo.Context, request []byte, response []byte) {
+			fmt.Printf("<<<< %s %s\n%s>>>> %d\n%s\n",
+				c.Request().Method,
+				c.Request().RequestURI,
+				string(request),
+				c.Response().Status,
+				string(response),
+			)
+		})
+	}
+
+	server.Use(logger)
 
 	// This group contains all of the Rosetta Data API endpoints.
 	server.POST("/network/list", dataCtrl.Networks)

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/onflow/flow-go v0.21.4
 	github.com/onflow/flow-go-sdk v0.21.0
 	github.com/onflow/flow-go/crypto v0.21.4
-	github.com/optakt/flow-dps v1.4.5
+	github.com/optakt/flow-dps v1.4.8
 	github.com/rs/zerolog v1.25.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0

go.sum

@@ -1037,8 +1037,8 @@ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/optakt/flow-dps v1.4.5 h1:Ma+cg1MDr06PAqYclLASKAhQI+hScet2ngGDBqazUdE=
-github.com/optakt/flow-dps v1.4.5/go.mod h1:hKyzEi9k9yj5OE8AeAV1c/g8tr5WokHKBk/L7Y1LZqk=
+github.com/optakt/flow-dps v1.4.8 h1:mmb+wMJvpJWrLD4hy/T1osKdEAQyABGvG6I5Z8HLZMQ=
+github.com/optakt/flow-dps v1.4.8/go.mod h1:hKyzEi9k9yj5OE8AeAV1c/g8tr5WokHKBk/L7Y1LZqk=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=