From 3652f119731fd48262f711a2da238cd36175f52b Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Tue, 28 Nov 2023 20:13:43 +0530 Subject: [PATCH] Update snapshotter state machine documentation --- docs/Protocol/Specifications/Epoch.md | 14 +- .../Specifications/Snapshotter/preloading.md | 11 +- .../Snapshotter/state_machine.md | 7 +- docs/Snapshotters/health-tracking.md | 135 +++++++++++++++++- docs/Specification/_category_.json | 8 -- docs/Specification/index.md | 5 - docusaurus.config.js | 1 - 7 files changed, 158 insertions(+), 23 deletions(-) delete mode 100644 docs/Specification/_category_.json delete mode 100644 docs/Specification/index.md diff --git a/docs/Protocol/Specifications/Epoch.md b/docs/Protocol/Specifications/Epoch.md index 4c8811d..93c6f00 100644 --- a/docs/Protocol/Specifications/Epoch.md +++ b/docs/Protocol/Specifications/Epoch.md @@ -31,23 +31,29 @@ For every `epochId`, the following are a sequence of states that a 'project' goe The snapshotter nodes participate in these states according to the data sources configured by the data markets they participate in and the status of the same can be diagnosed further with their internal APIs. -### `EPOCH_RELEASED` +### 1. `EPOCH_RELEASED` When an epoch is released on the protocol by the [epoch generator service](#epoch-generator). -### `PRELOAD` +### 2. `PRELOAD` For every project type's preloader configuration, this state is considered succesful once all the preloading dependencies are satisfied. -### `SNAPSHOT_BUILD` +### 3. `SNAPSHOT_BUILD` This state is considered successful when the snapshot data structure is computed and made ready for submission to the protocol for consensus. This applies to both base as well as aggregate snapshots. -### `SNAPSHOT_SUBMIT_PAYLOAD_COMMIT` +### 4. `SNAPSHOT_SUBMIT_PAYLOAD_COMMIT` This is an internal state specific to snapshotter implementations where the snapshot building worker processes, upon a successful `STATE_BUILD`, queue them up with the Payload Commit Service. +### 5. `RELAYER_SEND` +Payload commit service has sent the snapshot to a transaction relayer to submit to the protocol state contract. + +### 6. `SNAPSHOT_FINALIZE` + +Snapshot is finalized on the protocol. ## Useful links diff --git a/docs/Protocol/Specifications/Snapshotter/preloading.md b/docs/Protocol/Specifications/Snapshotter/preloading.md index 1bccfc8..ae4465f 100644 --- a/docs/Protocol/Specifications/Snapshotter/preloading.md +++ b/docs/Protocol/Specifications/Snapshotter/preloading.md @@ -2,4 +2,13 @@ sidebar_position: 1 --- -# Preloading \ No newline at end of file +# Preloading + +Preloaders perform an important function of fetching low-level data for eg. block details, and transaction receipts so that subsequent base snapshot building can proceed without performing unnecessary redundant calls that ultimately save on access costs on RPC and other queries on the underlying node infrastructure for the source data blockchain. + +Each project type within the project configuration as found in config/projects.json can specify the preloaders that their base snapshot builds depend on. Once the dependent preloaders have completed their fetches, the Processor Distributor subsequently triggers the base snapshot builders for each project type. + +```json reference +https://github.com/PowerLoom/pooler/blob/5e7cc3812074d91e8d7d85058554bb1175bf8070/config/projects.example.json#L3-L12 +``` + diff --git a/docs/Protocol/Specifications/Snapshotter/state_machine.md b/docs/Protocol/Specifications/Snapshotter/state_machine.md index 923ad43..54c729d 100644 --- a/docs/Protocol/Specifications/Snapshotter/state_machine.md +++ b/docs/Protocol/Specifications/Snapshotter/state_machine.md @@ -4,7 +4,8 @@ sidebar_position: 0 # State Machine -T - -These state transitions can be inspected in detail by querying the snapshotter node's internal APIs. +Refer to the following sections for detailed information on the state transitions that a snapshotter participates in while processing data sources per epoch. +* [Epoch processing state transitions](Protocol/Specifications/Epoch.md#state-transitions) +* [Snapshotter internal APIs](/docs/Snapshotters/health-tracking.md) + diff --git a/docs/Snapshotters/health-tracking.md b/docs/Snapshotters/health-tracking.md index 5d7070d..b4950e2 100644 --- a/docs/Snapshotters/health-tracking.md +++ b/docs/Snapshotters/health-tracking.md @@ -2,4 +2,137 @@ sidebar_position: 5 --- -# Health Tracking \ No newline at end of file +# Health Tracking + +All the snapshotter implementations include internal APIs that offer a detailed view on the state of processing of projects as configured by data sources, per epoch. + +You can tunnel into port `8002` of an instance running the snapshotter and right away try out the internal APIs among others by visting the FastAPI generated SwaggerUI. + +## `GET /internal/snapshotter/epochProcessingStatus` + +As detailed out in the section on [epoch processing state transitions](Protocol/Specifications/Epoch.md#state-transitions), this internal API endpoint offers the most detailed insight into each epoch's processing status as it passes through the snapshot builders and is sent out for consensus. + +> NOTE: The endpoint, though paginated and cached, serves a raw dump of insights into an epoch's state transitions and the payloads are significantly large enough for requests to timeout or to clog the internal API's limited resource. Hence it is advisable to query somewhere between 1 to 5 epochs. The same can be specified as the size query parameter. + +### Sample request + +```bash +curl -X 'GET' \ + 'http://localhost:8002/internal/snapshotter/epochProcessingStatus?page=1&size=3' \ + -H 'accept: application/json' +``` + +### Sample response + +```json +{ + "items": [ + { + "epochId": 43523, + "transitionStatus": { + "EPOCH_RELEASED": { + "status": "success", + "error": null, + "extra": null, + "timestamp": 1692530595 + }, + "PRELOAD": { + "pairContract_pair_total_reserves": { + "status": "success", + "error": null, + "extra": null, + "timestamp": 1692530595 + }, + }, + "SNAPSHOT_BUILD": { + "aggregate_24h_stats_lite:35ee1886fa4665255a0d0486c6079c4719c82f0f62ef9e96a98f26fde2e8a106:UNISWAPV2": { + "status": "success", + "error": null, + "extra": null, + "timestamp": 1692530596 + }, + }, + "SNAPSHOT_SUBMIT_PAYLOAD_COMMIT": { + + }, + "RELAYER_SEND": { + + }, + "SNAPSHOT_FINALIZE": { + + }, + }, + } + ], + "total": 3, + "page": 1, + "size": 3, + "pages": 1 +} +``` + +## `GET /internal/snapshotter/status` + +Returns the overall status of all the projects + +### Sample request + +```bash +curl -X 'GET' \ + 'http://localhost:8002/internal/snapshotter/status' \ + -H 'accept: application/json' +``` + +### Sample response + +```json +{ + "totalSuccessfulSubmissions": 10, + "totalMissedSubmissions": 5, + "totalIncorrectSubmissions": 1, + "projects":[ + { + "projectId": "projectid" + "successfulSubmissions": 3, + "missedSubmissions": 2, + "incorrectSubmissions": 1 + }, + ] +} +``` + +## `GET /internal/snapshotter/status/{project_id}` + +Returns project specific detailed status report with snapshot data + +### Sample request + +```bash +curl -X 'GET' \ + 'http://localhost:8002/internal/snapshotter/status/project1122' \ + -H 'accept: application/json' +``` + +### Sample response + +```json +{ + "missedSubmissions": [ + { + "epochId": 10, + "finalizedSnapshotCid": "cid", + "reason": "error/exception/trace" + } + ], + "incorrectSubmissions": [ + { + "epochId": 12, + "submittedSnapshotCid": "snapshotcid", + "submittedSnapshot": {} + "finalizedSnapshotCid": "finalizedsnapshotcid", + "finalizedSnapshot": {}, + "reason": "reason for incorrect submission" + } + ] +} +``` \ No newline at end of file diff --git a/docs/Specification/_category_.json b/docs/Specification/_category_.json deleted file mode 100644 index 6589297..0000000 --- a/docs/Specification/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Specification", - "position": 4, - "link": { - "type": "generated-index", - "description": "5 minutes to learn the most important Docusaurus concepts." - } - } \ No newline at end of file diff --git a/docs/Specification/index.md b/docs/Specification/index.md deleted file mode 100644 index d5c2669..0000000 --- a/docs/Specification/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -sidebar_position: 0 ---- - -# Core Concepts \ No newline at end of file diff --git a/docusaurus.config.js b/docusaurus.config.js index 5af20be..8b27f53 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -161,7 +161,6 @@ const config = { themes: [ 'docusaurus-theme-openapi-docs', '@saucelabs/theme-github-codeblock' - ] };