From e1c12ec4dd1b686282aba37b9fd0dae8bdb91250 Mon Sep 17 00:00:00 2001 From: "g. nicholas d'andrea" Date: Wed, 3 Jul 2024 16:43:03 -0400 Subject: [PATCH 1/2] Organize implementation guide a bit - Move summary/overview pages into their parent and remove corresponding category pages - Reword some things - Move the Jest framework code to the end of the testing section - Add titles to usage examples to reduce risk that these will be mistaken to be reference implementation code. --- .../implementation-guides/_category_.json | 8 -- .../implementation-guides.mdx | 57 +++++++++ .../pointers/_category_.json | 8 -- .../dereference-logic/dereference-logic.mdx | 120 ++++++++++++++++-- .../dereference-logic/generating-regions.mdx | 12 +- .../pointers/dereference-logic/summary.mdx | 108 ---------------- .../pointers/{overview.mdx => pointers.mdx} | 14 +- .../pointers/testing/_category_.json | 8 -- .../testing/blockchain-simulation.mdx | 2 +- .../pointers/testing/jest.mdx | 39 ++++++ .../pointers/testing/test-cases.mdx | 6 +- .../testing/{summary.mdx => testing.mdx} | 39 +----- .../pointers/types/pointer-types.mdx | 4 +- 13 files changed, 230 insertions(+), 195 deletions(-) delete mode 100644 packages/web/docs/implementation-guides/_category_.json create mode 100644 packages/web/docs/implementation-guides/implementation-guides.mdx delete mode 100644 packages/web/docs/implementation-guides/pointers/_category_.json delete mode 100644 packages/web/docs/implementation-guides/pointers/dereference-logic/summary.mdx rename packages/web/docs/implementation-guides/pointers/{overview.mdx => pointers.mdx} (83%) delete mode 100644 packages/web/docs/implementation-guides/pointers/testing/_category_.json create mode 100644 packages/web/docs/implementation-guides/pointers/testing/jest.mdx rename packages/web/docs/implementation-guides/pointers/testing/{summary.mdx => testing.mdx} (63%) diff --git a/packages/web/docs/implementation-guides/_category_.json b/packages/web/docs/implementation-guides/_category_.json deleted file mode 100644 index a8bbe734..00000000 --- a/packages/web/docs/implementation-guides/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Implementation guides", - "position": 4, - "link": { - "type": "generated-index", - "description": "Guides on how to implement ethdebug/format" - } -} diff --git a/packages/web/docs/implementation-guides/implementation-guides.mdx b/packages/web/docs/implementation-guides/implementation-guides.mdx new file mode 100644 index 00000000..4574f362 --- /dev/null +++ b/packages/web/docs/implementation-guides/implementation-guides.mdx @@ -0,0 +1,57 @@ +--- +sidebar_position: 4 +pagination_prev: null +pagination_next: null +--- + +# Implementation guides + +This section of these docs serves to provide resources that guide readers who +are looking to implement one or more components of **ethdebug/format**. +Because of the distinct concerns involved in implementing this format on the +compilation side vs. the concerns involved on the debugging side, this page +lists and categorizes the available guides into the appropriate heading. + + +## For debuggers + +
+
**Guide: [Dereferencing pointers](/docs/implementation-guides/pointers)**
+
+This guide provides readers with a tour of the **@ethdebug/pointers** +TypeScript reference implementation, showing example concrete logic for how a +debugger might process **ethdebug/format** pointers. + +For an introduction to **ethdebug/format** pointers, please see +the Pointer specification's [Overview](/spec/pointer/overview) and +[Key concepts](/spec/pointer/concepts) pages. +
+ +
**Other guides**
+
+_Guides for other aspects of debugger-side **ethdebug/format** implementation +are planned and still need to be written._ +
+ +
+ +## For compilers + +
+
**No availble guides yet**
+
+_Guides for implementing **ethdebug/format** support inside a compiler are +planned and still need to be written._ +
+
+ +:::tip[Work in progress] + +Sadly, things are looking a little scarce right now. Please stay tuned as work +on this effort progresses. + +**Interested in helping out?** If you'd like to help with writing initial +reference implementations for one or more schemas, please reach out in our +[Matrix.chat](https://matrix.to/#/#ethdebug:matrix.org). + +::: diff --git a/packages/web/docs/implementation-guides/pointers/_category_.json b/packages/web/docs/implementation-guides/pointers/_category_.json deleted file mode 100644 index 9fb79897..00000000 --- a/packages/web/docs/implementation-guides/pointers/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Dereferencing pointers", - "position": 2, - "link": { - "type": "generated-index", - "description": "Debugger-side reference implementation of ethdebug/format/pointer" - } -} diff --git a/packages/web/docs/implementation-guides/pointers/dereference-logic/dereference-logic.mdx b/packages/web/docs/implementation-guides/pointers/dereference-logic/dereference-logic.mdx index 647252a9..f6e706ba 100644 --- a/packages/web/docs/implementation-guides/pointers/dereference-logic/dereference-logic.mdx +++ b/packages/web/docs/implementation-guides/pointers/dereference-logic/dereference-logic.mdx @@ -1,20 +1,116 @@ --- sidebar_position: 5 +sidebar_label: The dereference function --- -# The dereference function +import CodeListing from "@site/src/components/CodeListing"; -These next few pages cover how the components described thus far are combined -to create the final `dereference(pointer: Pointer)` function. +# The `dereference()` function -- The [Summary](/docs/implementation-guides/pointers/dereference-logic/summary) - page broadly describes the control flow structure behind this function - implementation. +## Summary -- [Generating regions on the fly](/docs/implementation-guides/pointers/dereference-logic/generating-regions) - describes the process of recursively processing a pointer and reducing it to - a concrete list of fully-evaluated `Cursor.Region` objects. +The pages in this section cover the internals of the `dereference()` function +in the **@ethdebug/pointers** reference implementation. -- [Making regions concrete](/docs/implementation-guides/pointers/dereference-logic/making-regions-concrete) - describes the process for converting a single `Pointer.Region` object into - its fully-evaluated `Cursor.Region` equivalent at runtime. +The full signature of this function is as follows: + + { + const definition = sourceFile.getFunction("dereference"); + const tempSourceFile = project.createSourceFile( + "dereference-summary.ts", + "", + { overwrite: true } + ); + + for (const importDeclaration of sourceFile.getImportDeclarations()) { + tempSourceFile.addImportDeclaration(importDeclaration.getStructure()); + } + + const commentText = definition.getLeadingCommentRanges() + .map(range => + sourceFile.getFullText() + .substring(range.getPos(), range.getEnd())) + .join("\n"); + + const declaration = tempSourceFile.addFunction({ + name: definition.getName(), + parameters: definition.getParameters() + .map((param, index, array) => ({ + name: param.getName(), + type: param.getType().getText(param), + hasQuestionToken: param.hasQuestionToken() || param.hasInitializer(), + leadingTrivia: "\n", + trailingTrivia: index < array.length - 1 ? undefined : "\n" + })), + returnType: definition.getReturnType().getText(definition), + hasDeclareKeyword: true, + isAsync: true, + leadingTrivia: `${commentText}\n` + }); + + + return tempSourceFile.getFunction("dereference"); + } + } /> + +:::tip + +Remember from the +[Cursors](/docs/implementation-guides/pointers/types/cursors) section that a `Cursor` +provides a `view(state: Machine.State)` method, which returns an ordered +collection of concrete `Cursor.Region` objects. + +::: + +### `DereferenceOptions` + +Note the optional `options: DereferenceOptions` argument. This argument +allows for specifying additional information upfront that is necessary for +viewing the cursor later. Currently, this is needed only for pointers that +compose stack-located regions. + + sourceFile.getInterface("DereferenceOptions")} + /> + +## Control flow architecture + +The `dereference()` function itself performs two tasks: +1. Create a "simple cursor": a function that takes a machine state and + produces an asynchronous list of `Cursor.Region`s. +2. Adapt this simple cursor to conform to the full `Cursor` interface + +Within the process of creating this simple cursor it gets more interesting: +by leveraging JavaScript's +[AsyncIterator](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncIterator)s, +the implementation can compute regions on the fly by recursively breaking down +pointers into their nested child pointers. + +Since the desired end-result of `dereference()` is an object that can turn +a pointer into its composite ordered list of concrete regions at a particular +machine state, this implementation separates the concerns of generating this +list from converting this list into the promised return interface. + +To generate this list asynchronously on the fly, the implementation uses a +_stack of processing requests_ (which it calls "memos"), initially populated +with a request to dereference the root pointer. Each memo represents a state +or context change in some form: either a request to dereference a pointer or +sub-pointer, a request to save a set of regions by their names, or a request +to save the computed values of a set of variables by their identifiers. + +The other pages in this section proceed to go into more detail. + +
+See the full `src/dereference/index.ts` module + + +
diff --git a/packages/web/docs/implementation-guides/pointers/dereference-logic/generating-regions.mdx b/packages/web/docs/implementation-guides/pointers/dereference-logic/generating-regions.mdx index 331387c8..fa4b3dc3 100644 --- a/packages/web/docs/implementation-guides/pointers/dereference-logic/generating-regions.mdx +++ b/packages/web/docs/implementation-guides/pointers/dereference-logic/generating-regions.mdx @@ -147,7 +147,7 @@ so that earlier memos in the list will be processed before later ones. /> -### Processing a region +## Processing a region The simplest kind of pointer is just a single region. (Remember that pointers are either regions or collections of other pointers.) @@ -168,12 +168,12 @@ a request to save this region to process state by its name. This pointer evaluation process will be described later. -### Processing collections +## Processing collections The recursive cases are fairly straightforward following this architecture. -#### Groups +### Groups The simplest collection, a group of other pointers, yields no regions of its own, but instead pushes each of its child pointers for evaluation later: @@ -188,7 +188,7 @@ It's essential that each of the child pointers get evaluated in the order they appear in the list, since later pointers may reference regions named earlier, etc. -#### Lists +### Lists List collections are more complex because they dynamically generate a number of composed pointers based on a runtime count value and introducing a @@ -204,7 +204,7 @@ Note how, because each dynamic child pointer is evaluated based on the next incremented index value, the memos for updating this variable and evaluation the child pointer must be interspersed. -#### Conditionals +### Conditionals Conditional pointers evaluate to a child pointer given that some runtime condition evaluates to a nonzero value, optionally evaluating to a different @@ -220,7 +220,7 @@ the `"else"` pointer if it is specified: extract={sourceFile => sourceFile.getFunction("processConditional")} /> -#### Scopes +### Scopes Finally, the last kind of collection defined by this schema is for defining a scope of variables by identifier by specifying the expression values for diff --git a/packages/web/docs/implementation-guides/pointers/dereference-logic/summary.mdx b/packages/web/docs/implementation-guides/pointers/dereference-logic/summary.mdx deleted file mode 100644 index 69c665f7..00000000 --- a/packages/web/docs/implementation-guides/pointers/dereference-logic/summary.mdx +++ /dev/null @@ -1,108 +0,0 @@ ---- -sidebar_position: 1 ---- - -import CodeListing from "@site/src/components/CodeListing"; - -# Summary - -The pages in this section cover the internals of the `dereference()` function -implementation in the **@ethdebug/pointers** reference implementation -package. - -The full signature of this function is as follows: - - { - const definition = sourceFile.getFunction("dereference"); - const tempSourceFile = project.createSourceFile( - "dereference-summary.ts", - "", - { overwrite: true } - ); - - for (const importDeclaration of sourceFile.getImportDeclarations()) { - tempSourceFile.addImportDeclaration(importDeclaration.getStructure()); - } - - const commentText = definition.getLeadingCommentRanges() - .map(range => - sourceFile.getFullText() - .substring(range.getPos(), range.getEnd())) - .join("\n"); - - const declaration = tempSourceFile.addFunction({ - name: definition.getName(), - parameters: definition.getParameters() - .map((param, index, array) => ({ - name: param.getName(), - type: param.getType().getText(param), - hasQuestionToken: param.hasQuestionToken() || param.hasInitializer(), - leadingTrivia: "\n", - trailingTrivia: index < array.length - 1 ? undefined : "\n" - })), - returnType: definition.getReturnType().getText(definition), - hasDeclareKeyword: true, - isAsync: true, - leadingTrivia: `${commentText}\n` - }); - - - return tempSourceFile.getFunction("dereference"); - } - } /> - -**Remember** from the section on -[Cursors](/docs/implementation-guides/pointers/types/cursors) that a `Cursor` -provides a `view(state: Machine.State)` method, which returns an ordered -collection of concrete `Cursor.Region` objects. - -Note the optional `options: DereferenceOptions` argument. This argument -allows for specifying additional information upfront that is necessary for -viewing the cursor later. Currently, this is needed only for pointers that -compose stack-located regions. - - sourceFile.getInterface("DereferenceOptions")} - /> - -## Control flow architecture - -The `dereference()` function itself performs two tasks: -1. Create a "simple cursor": a function that takes a machine state and - produces an asynchronous list of `Cursor.Region`s. -2. Adapt this simple cursor to conform to the full `Cursor` interface - -Within the process of creating this simple cursor it gets more interesting: -by leveraging JavaScript's -[AsyncIterator](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncIterator)s, -the implementation can compute regions on the fly by recursively breaking down -pointers into their nested child pointers. - -Since the desired end-result of `dereference()` is an object that can turn -a pointer into its composite ordered list of concrete regions at a particular -machine state, this implementation separates the concerns of generating this -list from converting this list into the promised return interface. - -To generate this list asynchronously on the fly, the implementation uses a -_stack of processing requests_ (which it calls "memos"), initially populated -with a request to dereference the root pointer. Each memo represents a state -or context change in some form: either a request to dereference a pointer or -sub-pointer, a request to save a set of regions by their names, or a request -to save the computed values of a set of variables by their identifiers. - -The other pages in this section proceed to go into more detail. - -
-See the full `src/dereference/index.ts` module - - -
diff --git a/packages/web/docs/implementation-guides/pointers/overview.mdx b/packages/web/docs/implementation-guides/pointers/pointers.mdx similarity index 83% rename from packages/web/docs/implementation-guides/pointers/overview.mdx rename to packages/web/docs/implementation-guides/pointers/pointers.mdx index 54d83743..784deffa 100644 --- a/packages/web/docs/implementation-guides/pointers/overview.mdx +++ b/packages/web/docs/implementation-guides/pointers/pointers.mdx @@ -1,11 +1,21 @@ --- sidebar_position: 1 +sidebar_label: Dereferencing pointers --- import CodeBlock from "@theme/CodeBlock"; import CodeListing from "@site/src/components/CodeListing"; -# Overview +# Dereferencing pointers + +_An implementation guide for resolving **ethdebug/format/pointer**s +into concrete regions and values in a running EVM._ + +## Introduction + +This implementation guide describes the **@ethdebug/pointers** reference +implementation, written in TypeScript and intended for distribution as an +NPM package. Debuggers looking to support **ethdebug/format** must be prepared to dereference the data allocations used by compilers for all kinds of variables. @@ -17,8 +27,6 @@ This format's [**ethdebug/format/pointer** schema](/spec/pointer/overview) provides such suitable expressiveness, but implementing the logic to read and evaluate data in this schema requires some careful consideration. -This documentation section describes a reference implementation written in -TypeScript and available as the **@ethdebug/pointers** NPM package. :::tip If you're reading this page without first having familiarized yourself with the diff --git a/packages/web/docs/implementation-guides/pointers/testing/_category_.json b/packages/web/docs/implementation-guides/pointers/testing/_category_.json deleted file mode 100644 index f8c6ccd9..00000000 --- a/packages/web/docs/implementation-guides/pointers/testing/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "End-to-end testing", - "position": 6, - "link": { - "type": "generated-index", - "description": "Implementing integration tests" - } -} diff --git a/packages/web/docs/implementation-guides/pointers/testing/blockchain-simulation.mdx b/packages/web/docs/implementation-guides/pointers/testing/blockchain-simulation.mdx index 1f716288..6e772fda 100644 --- a/packages/web/docs/implementation-guides/pointers/testing/blockchain-simulation.mdx +++ b/packages/web/docs/implementation-guides/pointers/testing/blockchain-simulation.mdx @@ -9,7 +9,7 @@ import CodeListing from "@site/src/components/CodeListing"; :::warning In case you missed the -[note on the Summary page](/docs/implementation-guides/pointers/testing/summary#ganache-warning), +[note on this section's first page](/docs/implementation-guides/pointers/testing/#ganache-warning), the functionality described in this page uses the unmaintained [Ganache](https://github.com/trufflesuite/ganache) software library for simulating the EVM. See note for rationale and risk expectations. diff --git a/packages/web/docs/implementation-guides/pointers/testing/jest.mdx b/packages/web/docs/implementation-guides/pointers/testing/jest.mdx new file mode 100644 index 00000000..38751f93 --- /dev/null +++ b/packages/web/docs/implementation-guides/pointers/testing/jest.mdx @@ -0,0 +1,39 @@ +--- +sidebar_position: 8 +--- + +import { SyntaxKind, VariableDeclarationKind } from "ts-morph"; +import CodeListing from "@site/src/components/CodeListing"; + +# Hooking up Jest + +Putting all the pieces together, the following code listing shows the top-level +test implementation. These tests are written to use the +[Jest](https://jestjs.io/) testing framework. + +Notice the use of the `observeTraceTests` variable (from the previous +[Test cases](/docs/implementation-guides/pointers/testing/test-cases) page) +enables executing each test case without changing the core logic. + + { + const [describe] = sourceFile.getStatements() + .filter(statement => { + const callExpressions = statement.getChildrenOfKind(SyntaxKind.CallExpression); + + if (callExpressions.length === 0) { + return false; + } + + const functionName = callExpressions[0].getChildren()[0].getText(); + + return functionName === "describe"; + }); + + return describe; + } + +} /> diff --git a/packages/web/docs/implementation-guides/pointers/testing/test-cases.mdx b/packages/web/docs/implementation-guides/pointers/testing/test-cases.mdx index c7da154b..5e3d61c1 100644 --- a/packages/web/docs/implementation-guides/pointers/testing/test-cases.mdx +++ b/packages/web/docs/implementation-guides/pointers/testing/test-cases.mdx @@ -18,9 +18,9 @@ Test cases are aggregated into the `observeTraceTests` variable: extract={sourceFile => sourceFile.getVariableStatement("observeTraceTests")} /> -See the [Test code](/docs/implementation-guides/pointers/testing/summary#test-code) -heading on this section's summary page for how this variable is used by the -[Jest](https://jestjs.io) testing framework. +This variable will be used to generate automated tests dynamically, as will +be described on the +[next page](/docs/implementation-guides/pointers/testing/jest). ## Storage strings diff --git a/packages/web/docs/implementation-guides/pointers/testing/summary.mdx b/packages/web/docs/implementation-guides/pointers/testing/testing.mdx similarity index 63% rename from packages/web/docs/implementation-guides/pointers/testing/summary.mdx rename to packages/web/docs/implementation-guides/pointers/testing/testing.mdx index f858c2ae..2b32b962 100644 --- a/packages/web/docs/implementation-guides/pointers/testing/summary.mdx +++ b/packages/web/docs/implementation-guides/pointers/testing/testing.mdx @@ -1,11 +1,10 @@ --- -sidebar_position: 1 +sidebar_position: 6 --- -import { SyntaxKind, VariableDeclarationKind } from "ts-morph"; -import CodeListing from "@site/src/components/CodeListing"; +# End-to-end testing -# Summary +## Summary In addition to unit tests for each of the various modules, the **@ethdebug/pointers** reference implementation includes automated integration @@ -27,38 +26,6 @@ See more details [below](#ganache-warning). ::: -## Test code - -The following code listing shows the top-level test implementation. These tests -are written to use the [Jest](https://jestjs.io/) testing framework. - - { - const [describe] = sourceFile.getStatements() - .filter(statement => { - const callExpressions = statement.getChildrenOfKind(SyntaxKind.CallExpression); - - if (callExpressions.length === 0) { - return false; - } - - const functionName = callExpressions[0].getChildren()[0].getText(); - - return functionName === "describe"; - }); - - return describe; - } - -} /> - -In this code listing, please see how tests are generated programmatically based -on data definitions of each test case. (Notice how the `observeTraceTests` -variable is used.) - ## Unmaintained dependencies warning {#ganache-warning} The testing implementation described in this section uses the unmaintained diff --git a/packages/web/docs/implementation-guides/pointers/types/pointer-types.mdx b/packages/web/docs/implementation-guides/pointers/types/pointer-types.mdx index ca82db3c..afe03cfc 100644 --- a/packages/web/docs/implementation-guides/pointers/types/pointer-types.mdx +++ b/packages/web/docs/implementation-guides/pointers/types/pointer-types.mdx @@ -21,7 +21,7 @@ different kind of region and collection. A [full source listing](#code-listing) follows below, but see example usage to get a sense for how these types are organized: -```typescript +```typescript title="Usage example" import { Pointer, isPointer } from "@ethdebug/pointers"; const region: Pointer.Region = { location: "stack", slot: 0 }; @@ -42,7 +42,7 @@ nested, also roughly to correspond to the root JSON-Schema. See these quick examples to get a sense for this part of the type hierarchy: -```typescript +```typescript title="Usage example" import { Pointer } from "@ethdebug/pointers"; const expression: Pointer.Expression = { From 3766ecbd9ffdc1a79b02aa71ae54a819b3e0ebb5 Mon Sep 17 00:00:00 2001 From: "g. nicholas d'andrea" Date: Wed, 3 Jul 2024 16:46:03 -0400 Subject: [PATCH 2/2] Prevent external links' icon from wrapping itself --- packages/web/src/theme/MDXComponents.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/web/src/theme/MDXComponents.tsx b/packages/web/src/theme/MDXComponents.tsx index 52769443..3adeae13 100644 --- a/packages/web/src/theme/MDXComponents.tsx +++ b/packages/web/src/theme/MDXComponents.tsx @@ -13,13 +13,13 @@ library.add(fab, fas); // Add all icons to the library so you can use them witho // custom link component that appends external link svg inside Markdown files function A({ children, href, ...props }: LinkProps): JSX.Element { - return - {children} - {href && !isInternalUrl(href) && ( - - )} - + return { + children + }{href && !isInternalUrl(href) && ( + <> + ) }; } + export default { // Re-use the default mapping ...MDXComponents,