Merge branch 'master' into mraszyk/node-metrics

.github/workflows/pr-cleanup.yml

@@ -0,0 +1,35 @@
+name: PR Cleanup
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  release_preview_canister:
+    # do not run in forks
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.ref }}
+      cancel-in-progress: true
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - run: |
+          pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
+          pip install ic-py
+          python3 .github/workflows/scripts/release-canister.py $pull_number
+        env:
+          DFX_IDENTITY_PREVIEW: ${{ secrets.DFX_IDENTITY_PREVIEW }}
+          POOL_CANISTER_ID: ${{ secrets.POOL_CANISTER_ID }}
+
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            const comments = require('./.github/workflows/scripts/comments.js');
+            const maybeComment = await comments.get(context, github);
+            if (maybeComment) {
+              await comments.delete(context, github, maybeComment.id);
+            }

.github/workflows/preview-deployment.yml

@@ -0,0 +1,92 @@
+name: PR Preview Deployment
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  build_and_deploy:
+    # do not run in forks
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.ref }}
+      cancel-in-progress: true
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            const comments = require('./.github/workflows/scripts/comments.js');
+            const maybeComment = await comments.get(context, github);
+            if (maybeComment) {
+              await comments.update(context, github, maybeComment.id,  `🤖 Your PR preview is being built...`);
+            } else {
+              await comments.create(context, github, `🤖 Your PR preview is being built...`);
+            }
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install DFX
+        run: sh -ci "$(curl -fsSL https://internetcomputer.org/install.sh)"
+
+      - name: "Build & Deploy"
+        run: |
+          mkdir -p ~/.config/dfx/identity/default
+
+          echo $DFX_IDENTITY_PREVIEW | tr -d '\n\r ' | base64 -d > ~/.config/dfx/identity/default/identity.pem
+          sed -i 's/\\r\\n/\r\n/g' ~/.config/dfx/identity/default/identity.pem
+
+          # request preview canister from the pool
+          pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
+          pip install ic-py
+          canister_id=$(python3 .github/workflows/scripts/request-canister.py $pull_number)
+
+          # this is where the root of the web pages resides
+          cd docusaurus
+
+          # overwrite canister id in dfx.json
+          echo "{\"portal\":{\"ic\":\"$canister_id\"}}" > canister_ids.json
+
+          echo "PREVIEW_CANISTER_ID=$canister_id" >> $GITHUB_ENV
+
+          # install and deploy
+          npm install
+
+          export PREVIEW_CANISTER_ID=$canister_id
+
+          dfx deploy --network=ic --no-wallet portal --yes
+
+        env:
+          DFX_IDENTITY_PREVIEW: ${{ secrets.DFX_IDENTITY_PREVIEW }}
+          POOL_CANISTER_ID: ${{ secrets.POOL_CANISTER_ID }}
+
+      - name: Report build error
+        uses: actions/github-script@v6
+        if: ${{ failure() }}
+        with:
+          script: |
+            const comments = require('./.github/workflows/scripts/comments.js');
+            const maybeComment = await comments.get(context, github);
+            if (maybeComment) {
+              await comments.update(context, github, maybeComment.id,  `🤖 Preview build failed.`);
+            } else {
+              await comments.create(context, github, `🤖 Preview build failed.`);
+            }
+
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            const comments = require('./.github/workflows/scripts/comments.js');
+            const maybeComment = await comments.get(context, github);
+            if (maybeComment) {
+              await comments.update(context, github, maybeComment.id,  `🤖 Here's your preview: https://${process.env.PREVIEW_CANISTER_ID}.icp0.io/docs`);
+            } else {
+              await comments.create(context, github, `🤖 Here's your preview: https://${process.env.PREVIEW_CANISTER_ID}.icp0.io/docs`);
+            }

.github/workflows/scripts/comments.js

@@ -0,0 +1,37 @@
+exports.get = async function (context, github) {
+  const comments = await github.rest.issues.listComments({
+    issue_number: context.issue.number,
+    repo: context.repo.repo,
+    owner: context.repo.owner,
+  });
+
+  return comments.data.find(
+    (c) => c.user.login === 'github-actions[bot]' && c.user.type === 'Bot'
+  );
+};
+
+exports.create = function (context, github, body) {
+  return github.rest.issues.createComment({
+    issue_number: context.issue.number,
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    body,
+  });
+};
+
+exports.update = function (context, github, id, body) {
+  return github.rest.issues.updateComment({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    comment_id: id,
+    body,
+  });
+};
+
+exports.delete = function (context, github, id) {
+  return github.rest.issues.deleteComment({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    comment_id: id,
+  });
+};

.github/workflows/scripts/pool.py

@@ -0,0 +1,30 @@
+from ic.client import Client
+from ic.identity import Identity
+from ic.agent import Agent
+import os
+import sys
+import base64
+from ic.candid import encode, Types
+
+
+#
+# Interact with preview canister pool: https://github.com/dfinity/preview-canister-pool
+#
+
+private_key = base64.b64decode(os.environ["DFX_IDENTITY_PREVIEW"]).decode("utf-8")
+pool_id = os.environ["POOL_CANISTER_ID"]
+
+identity = Identity.from_pem(private_key)
+client = Client()
+agent = Agent(identity, client)
+
+def release_canister():
+  res = agent.update_raw(
+      pool_id, "release_canister", encode([{'type': Types.Text, 'value': sys.argv[1]}]))
+  return res
+
+
+def request_canister():
+  res = agent.update_raw(
+      pool_id, "request_canister", encode([{'type': Types.Text, 'value': sys.argv[1]}]), Types.Principal)
+  return res

.github/workflows/scripts/release-canister.py

@@ -0,0 +1,16 @@
+import os
+import sys
+
+if len(sys.argv) != 2:
+  print("Usage: python3 release-canister.py <ref>")
+  exit(1)
+
+for v in ["DFX_IDENTITY_PREVIEW","POOL_CANISTER_ID"]:
+  if not v in os.environ:
+    print(f"release-canister.py: {v} env variable missing")
+    exit(1)
+
+
+from pool import release_canister
+
+release_canister()

.github/workflows/scripts/request-canister.py

@@ -0,0 +1,16 @@
+import os
+import sys
+
+if len(sys.argv) != 2:
+  print("Usage: python3 request_canister.py <ref>")
+  exit(1)
+
+for v in ["DFX_IDENTITY_PREVIEW","POOL_CANISTER_ID"]:
+  if not v in os.environ:
+    print(f"request-canister.py: {v} env variable missing")
+    exit(1)
+
+from pool import request_canister
+
+canister_id = request_canister()[0]['value'].to_str()
+print(canister_id)

docusaurus/dfx.json

@@ -0,0 +1,32 @@
+{
+  "version": 1,
+  "canisters": {
+    "portal": {
+      "portal": {
+        "entrypoint": "build/index.html"
+      },
+      "source": [
+        "build/"
+      ],
+      "type": "assets"
+    }
+  },
+  "defaults": {
+    "build": {
+      "packtool": "",
+      "args": ""
+    }
+  },
+  "networks": {
+    "local": {
+      "bind": "127.0.0.1:8000",
+      "type": "ephemeral"
+    },
+    "ic": {
+      "providers": [
+        "https://mainnet.dfinity.network"
+      ],
+      "type": "persistent"
+    }
+  }
+}

spec/_attachments/interface-spec-changelog.md

@@ -5,6 +5,7 @@
 * System API calls `ic0.msg_caller_size` and `ic0.msg_caller_copy` can be called in all contexts except for (start) function.
 * Added note on confidentiality of values in the certified state tree.
 * Update algorithm computing the request and response hash in the HTTP Gateway including clarification of when the HTTP Gateway can allow for arbitrary certification version in the canister's response.
+* Update conditions on requested paths in HTTP read state requests.
 
 ### 0.20.0 (2023-07-11) {#0_20_0}
 * IC Bitcoin API, ECDSA API, canister HTTPS outcalls API, and 128-bit cycles System API are considered stable.

spec/http-gateway-protocol-spec.md

@@ -375,10 +375,18 @@ The steps for response verification are as follows:
 
 ## Response Verification Version Assertion
 
-Canisters can report the versions of response verification that they support using public metadata in the [system state tree](https://internetcomputer.org/docs/current/references/ic-interface-spec/#state-tree-canister-information). This metadata will be read by the HTTP Gateway using a [read_state request](https://internetcomputer.org/docs/current/references/ic-interface-spec/#http-read-state). This metadata is a comma-delimited string of versions under the key "supported_certificate_versions”, for example: "1,2". This is treated as an optional, additional layer of security for canisters supporting multiple versions. If the metadata has not been added (i.e. the lookup of this metadata in the `read_state` response returns `Absent`), then the HTTP Gateway will allow for whatever version the canister has responded with.
+Canisters can report the supported versions of response verification using (public) metadata sections available in the [system state tree](https://internetcomputer.org/docs/current/references/ic-interface-spec/#state-tree-canister-information). This metadata will be read by the HTTP Gateway using a [read_state request](https://internetcomputer.org/docs/current/references/ic-interface-spec/#http-read-state). The metadata section must be a (public) custom section with the name `supported_certificate_versions` and contain a comma-delimited string of versions, e.g., `1,2`. This is treated as an optional, additional layer of security for canisters supporting multiple versions. If the metadata has not been added (i.e., the `read_state` request *succeeds* and the lookup of the metadata section in the `read_state` response certificate returns `Absent`), then the HTTP Gateway will allow for whatever version the canister has responded with.
+
 
 The request for the metadata will only be made by the HTTP Gateway if there is a downgrade. If the HTTP Gateway requests v2 and the canister responds with v2, then a request will not be made. If the HTTP Gateway requests v2 and the canister responds with v1, a request will be made. If a request is made, the HTTP Gateway will not accept any response from the canister that is below the max version supported by both the HTTP Gateway and the canister. This will guarantee that a canister supporting both v1 and v2 will always have v2 security when accessed by an HTTP Gateway that supports v2.
 
+:::note
+
+The HTTP Gateway can only allow for arbitrary certification version if the custom section `supported_certificate_versions` is *provably* not present, i.e., if the `read_state` response contains a valid certificate whose lookup of the corresponding path yields `Absent`. Otherwise, e.g., if the replica is overloaded or if the `read_state` is rejected because the custom section with the name `supported_certificate_versions` is private, the HTTP Gateway should also reject the canister's response.
+
+:::
+
+
 ## Canister HTTP Interface
 
 The full [Candid](https://github.com/dfinity/candid/blob/master/spec/Candid.md) interface that a canister is expected to implement is as follows: