From b46aef2a9b8fe4f0e1732bc9ea5127dd436d6572 Mon Sep 17 00:00:00 2001 From: Maks Pawlak <120831+m4ksio@users.noreply.github.com> Date: Fri, 24 Sep 2021 19:16:55 -0700 Subject: [PATCH 1/4] Add simple wait mode --- cmd/flow-rosetta-server/main.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/flow-rosetta-server/main.go b/cmd/flow-rosetta-server/main.go index 8365e12..08a0fc4 100755 --- a/cmd/flow-rosetta-server/main.go +++ b/cmd/flow-rosetta-server/main.go @@ -69,6 +69,7 @@ func run() int { flagPort uint16 flagTransactions uint flagSmart bool + flagWait bool ) pflag.StringVarP(&flagDPS, "dps-api", "a", "127.0.0.1:5005", "host address for GRPC API endpoint") @@ -78,6 +79,7 @@ func run() int { pflag.Uint16VarP(&flagPort, "port", "p", 8080, "port to host Rosetta API on") pflag.UintVarP(&flagTransactions, "transaction-limit", "t", 200, "maximum amount of transactions to include in a block response") pflag.BoolVar(&flagSmart, "smart-status-codes", false, "enable smart non-500 HTTP status codes for Rosetta API errors") + pflag.BoolVarP(&flagWait, "wait-for-index", "w", false, "wait for index to be available instead of quitting right away, useful when DPS Live index bootstraps") pflag.Parse() @@ -105,10 +107,15 @@ func run() int { dpsAPI := api.NewAPIClient(conn) index := api.IndexFromAPI(dpsAPI, codec) +wait: // Deduce chain ID from DPS API to configure parameters for script exec. first, err := index.First() if err != nil { log.Error().Err(err).Msg("could not get first height from DPS API") + if flagWait { + time.Sleep(30 * time.Second) + goto wait + } return failure } root, err := index.Header(first) From 437c0849089352beff9291015669afb4b6b22b88 Mon Sep 17 00:00:00 2001 From: Maks Pawlak <120831+m4ksio@users.noreply.github.com> Date: Thu, 7 Oct 2021 01:43:49 -0700 Subject: [PATCH 2/4] Add debug requests mode --- cmd/flow-rosetta-server/main.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/cmd/flow-rosetta-server/main.go b/cmd/flow-rosetta-server/main.go index 08a0fc4..65d7474 100755 --- a/cmd/flow-rosetta-server/main.go +++ b/cmd/flow-rosetta-server/main.go @@ -24,6 +24,7 @@ import ( "time" "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" "github.com/rs/zerolog" "github.com/spf13/pflag" "github.com/ziflex/lecho/v2" @@ -70,6 +71,7 @@ func run() int { flagTransactions uint flagSmart bool flagWait bool + flagDump bool ) pflag.StringVarP(&flagDPS, "dps-api", "a", "127.0.0.1:5005", "host address for GRPC API endpoint") @@ -79,6 +81,7 @@ func run() int { pflag.Uint16VarP(&flagPort, "port", "p", 8080, "port to host Rosetta API on") pflag.UintVarP(&flagTransactions, "transaction-limit", "t", 200, "maximum amount of transactions to include in a block response") pflag.BoolVar(&flagSmart, "smart-status-codes", false, "enable smart non-500 HTTP status codes for Rosetta API errors") + pflag.BoolVar(&flagDump, "dump-requests", false, "print out full request and responses") pflag.BoolVarP(&flagWait, "wait-for-index", "w", false, "wait for index to be available instead of quitting right away, useful when DPS Live index bootstraps") pflag.Parse() @@ -176,7 +179,22 @@ wait: server.HideBanner = true server.HidePort = true server.Logger = elog - server.Use(lecho.Middleware(lecho.Config{Logger: elog})) + + logger := lecho.Middleware(lecho.Config{Logger: elog}) + + if flagDump { + logger = middleware.BodyDump(func(c echo.Context, request []byte, response []byte) { + fmt.Printf("<<<< %s %s\n%s>>>> %d\n%s\n", + c.Request().Method, + c.Request().RequestURI, + string(request), + c.Response().Status, + string(response), + ) + }) + } + + server.Use(logger) // This group contains all of the Rosetta Data API endpoints. server.POST("/network/list", dataCtrl.Networks) From 4009bfd572b445fad2be5c0936136ba550578bfe Mon Sep 17 00:00:00 2001 From: Chris Haen Date: Thu, 17 Mar 2022 10:07:44 -0400 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=8C=8AUpdates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- README.md | 98 +++++++++++++++++++++++++++++++ cmd/flow-rosetta-server/flow.json | 2 +- go.mod | 4 +- go.sum | 4 +- 5 files changed, 105 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index c56174b..e9efd6c 100755 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.cdc *.checkpoint *.log -flow-go/ \ No newline at end of file +flow-go/ +.vscode/ \ No newline at end of file diff --git a/README.md b/README.md index 103d7d4..aab06ee 100755 --- a/README.md +++ b/README.md @@ -1 +1,99 @@ # flow-rosetta +forked from [dapperlabs/flow-rosetta](https://github.com/dapperlabs/flow-rosetta) + +## Description + +The Flow Rosetta Server implements the Rosetta Data API specifications for the Flow network. +It uses the Flow DPS Server's GRPC API as the backend to query the required data. +Flow core contract addresses are derived from the chain ID with which the service is started. +This allows the Rosetta API to access state remotely, or locally by running the Flow DPS Server on the same host. + +## Dependencies + +Go `v1.16` or higher is required to compile `flow-dps-rosetta`. +Only Linux amd64 builds are supported, because of the dependency to the [`flow-go/crypto`](https://github.com/onflow/flow-go/tree/master/crypto) package. + +In order to build the live binary, the following extra steps and dependencies are required: + +* [`CMake`](https://cmake.org/install/) + +Please note that the flow-go repository should be cloned into this projects directory with its default name, so that the Go module replace statement works as intended: `replace github.com/onflow/flow-go/crypto => ./flow-go/crypto`. + +* `git clone git@github.com:onflow/flow-go.git` +* `cd flow-go/crypto` +* `git checkout v0.21.4` +* `go generate` + +You can then verify that the installation of the flow-go crypto package has been successful by running the tests of the project. + +## Testing + +The Flow system model, where resources can be moved freely between accounts without generating events, makes it impossible to fully reconcile account balances on the Rosetta Data API. +As a consequence, balance reconciliation has to be disabled when running the Rosetta CLI against the Flow Rosetta Server. + +Additionally, some of the events generated by Flow are not accurately reflecting the account address they relate to. +This is also due to the same resource-based smart contract programming model. +For instance, when an account receives vaults from different locations to execute a swap of tokens, the events related to this swap might indicate the swap +contract's address, as it uses volatile vaults. + +Currently, the only way to work around this issue is to create exemptions for accounts which contain such smart contracts. +As account balances using non-standard approaches to transfer Flow tokens can already not be reconciled, this is an acceptable limitation. +In general, transactions by Rosetta should not be used to deduce account balances. +Full historical account balance lookup is available and should thus be prefered to determine the account balance at any block height. + +The discussed configuration is available in the `flow.json` and `exemptions.json` files for the `mainnet-9` spork DPS. +The following command can be executed to validate the Data API for that spork: + +```sh +rosetta-cli check:data --configuration-file flow.json +``` + +## Usage + +```sh +Usage of flow-rosetta-server: + -a, --api string host URL for GRPC API endpoint (default "127.0.0.1:5005") + -e, --cache uint maximum cache size for register reads in bytes (default 1073741824) + -l, --level string log output level (default "info") + -p, --port uint16 port to host Rosetta API on (default 8080) + -t, --transaction-limit int maximum amount of transactions to include in a block response (default 200) + --smart-status-codes enable smart non-500 HTTP status codes for Rosetta API errors +``` + +## Example + +The following command line starts the Flow Rosetta server for a main network spork on port 8080. +It uses a local instance of the Flow DPS Server for access to the execution state. + +```sh +./flow-rosetta-server -a "127.0.0.1:5005" -p 8080 +``` + +## Architecture + +The Rosetta API needs its own documentation because of the amount of components it has that interact with each other. +The main reason for its complexity is that it needs to interact with the Flow Virtual Machine (FVM) and to translate between the Flow and Rosetta application domains. + +### Invoker + +This component, given a Cadence script, can execute it at any given height and return the value produced by the script. + +[Package documentation](https://pkg.go.dev/github.com/optakt/flow-dps-rosetta/service/invoker) + +### Retriever + +The retriever uses the other components to retrieve account balances, blocks and transactions. + +[Package documentation](https://pkg.go.dev/github.com/optakt/flow-dps-rosetta/service/retriever) + +### Scripts + +The script package produces Cadence scripts with the correct imports and storage paths, depending on the configured Flow chain ID. + +[Package documentation](https://pkg.go.dev/github.com/optakt/flow-dps-rosetta/service/scripts) + +### Validator + +The Validator component validates whether the given Rosetta identifiers are valid. + +[Package documentation](https://pkg.go.dev/github.com/optakt/flow-dps-rosetta/service/validator) diff --git a/cmd/flow-rosetta-server/flow.json b/cmd/flow-rosetta-server/flow.json index 5876f48..014b41c 100755 --- a/cmd/flow-rosetta-server/flow.json +++ b/cmd/flow-rosetta-server/flow.json @@ -30,7 +30,7 @@ "interesting_accounts": "", "reconciliation_disabled": true, "reconciliation_drain_disabled": false, - "inactive_discrepency_search_disabled": false, + "inactive_discrepancy_search_disabled": false, "balance_tracking_disabled": false, "coin_tracking_disabled": false, "status_port": 9090, diff --git a/go.mod b/go.mod index cc63027..f11ee86 100755 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/onflow/cadence v0.19.1 github.com/onflow/flow-go v0.21.2 github.com/onflow/flow-go-sdk v0.21.0 - github.com/onflow/flow-go/crypto v0.21.2 - github.com/optakt/flow-dps v1.3.3 + github.com/onflow/flow-go/crypto v0.21.4 + github.com/optakt/flow-dps v1.4.8 github.com/rs/zerolog v1.25.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 3e5d246..feb90dc 100644 --- a/go.sum +++ b/go.sum @@ -1037,8 +1037,8 @@ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/optakt/flow-dps v1.3.3 h1:JJDAXdFZEEShde7KhS8muGYv6OPdeHjhGRnW+iMFrkM= -github.com/optakt/flow-dps v1.3.3/go.mod h1:/J6EnGu0OyBx6+xF2wnSaeN65J4LVaIKgUfImyTXVDs= +github.com/optakt/flow-dps v1.4.8 h1:mmb+wMJvpJWrLD4hy/T1osKdEAQyABGvG6I5Z8HLZMQ= +github.com/optakt/flow-dps v1.4.8/go.mod h1:hKyzEi9k9yj5OE8AeAV1c/g8tr5WokHKBk/L7Y1LZqk= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= From a9f6ac5836324a0fafa1042d898c75dccc6bc06e Mon Sep 17 00:00:00 2001 From: Chris Haen Date: Thu, 17 Mar 2022 10:08:18 -0400 Subject: [PATCH 4/4] mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f11ee86..1fc8282 100755 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/klauspost/compress v1.13.5 github.com/labstack/echo/v4 v4.5.0 github.com/onflow/cadence v0.19.1 - github.com/onflow/flow-go v0.21.2 + github.com/onflow/flow-go v0.21.4 github.com/onflow/flow-go-sdk v0.21.0 github.com/onflow/flow-go/crypto v0.21.4 github.com/optakt/flow-dps v1.4.8 diff --git a/go.sum b/go.sum index feb90dc..5030062 100644 --- a/go.sum +++ b/go.sum @@ -1000,8 +1000,8 @@ github.com/onflow/flow-emulator v0.20.3/go.mod h1:xNdVsrMJiAaYJ59Dwo+xvj0ENdvk/b github.com/onflow/flow-ft/lib/go/contracts v0.5.0 h1:Cg4gHGVblxcejfNNG5Mfj98Wf4zbY76O0Y28QB0766A= github.com/onflow/flow-ft/lib/go/contracts v0.5.0/go.mod h1:1zoTjp1KzNnOPkyqKmWKerUyf0gciw+e6tAEt0Ks3JE= github.com/onflow/flow-go v0.18.0/go.mod h1:cQpvFoqth9PR7tarWDa36R/dDOqUK5QYfeYzCdXPLII= -github.com/onflow/flow-go v0.21.2 h1:p6mfz4n7ImPytYVYRn9Zcx56fHEd8LCRTK4YdMWVeyk= -github.com/onflow/flow-go v0.21.2/go.mod h1:1greKu6cK7okw4gvs8qSuKSp7W5JOPGKxJvTRjraY5s= +github.com/onflow/flow-go v0.21.4 h1:S7KLt0nJmFUuc3rX7aqYcdlWGHE6z+hpr2cW8KQO3q8= +github.com/onflow/flow-go v0.21.4/go.mod h1:1greKu6cK7okw4gvs8qSuKSp7W5JOPGKxJvTRjraY5s= github.com/onflow/flow-go-sdk v0.20.0-alpha.1/go.mod h1:52QZyLwU3p3UZ2FXOy+sRl4JPdtvJoae1spIUBOFxA8= github.com/onflow/flow-go-sdk v0.20.0/go.mod h1:52QZyLwU3p3UZ2FXOy+sRl4JPdtvJoae1spIUBOFxA8= github.com/onflow/flow-go-sdk v0.21.0 h1:KRU6F80KZLD+CJLj57S2EaAPNJUx4qpFTw1Ok0AJZ1M=