feat!: Re-enable interface tests and get them succeeding (#83)

* chore: sync with ipfs/test/interface-http-go.js

see: https://github.com/ipfs/js-ipfs/blob/6ae5eb7da33adfdee70c7edc09a0f644d5347718/packages/ipfs/test/interface-http-go.js

* chore: install multiple go-ipfs versions

`ipfs` uses `[email protected]`, which means the tests we copied will not
work the same.
See https://github.com/ipfs/js-ipfs/blob/6ae5eb7/packages/ipfs/package.json#L93

Adding multiple go versions in this commit. Will be facilitating the
use of these versions in the next commit

* fix: .bitswap.wantlist before test

* fix: .bitswap.wantlistForPeer before

* chore: enable running interface tests against multiple kubo versions

* fix: bitswap ipfs spawn types

* fix: dht utils ipfs api usage

* fix: dag.get test

* fix: dht tests

* fix: .id test

* fix: pinning service tests

* fix: misc tests

* fix: pinning tests

* fix: .ping tests

* fix: .swarm.peers passes

* fix: .swarm.addrs passes

* fix: .swarm.disconnect passes

* fix: all tests are passing in node

some additional tests were skipped, but node tests are succeeding

* fix: browser interface tests are passing or skipped

* fix: webworker tests pass

* feat: fix webworker tests

* chore: code cleanup

* chore: commentary and code cleanup

* fix: update package-lock.json

* chore: update current go-ipfs to v0.17

* chore: 'should compress a directory as a tarball' actually passes

* Revert "chore: 'should compress a directory as a tarball' actually passes"

This reverts commit 1efca00.

* use lts on test node

* fix: .pubsub.subscribe tests

* chore: actually wait for factory cleanup

* chore: fix lint and package-lock

* fix: dep-check

* chore: skip .refs tests

@see #77

* chore: remove root tests not implemented in kubo

* chore: add lint:fix script

* chore: consolidate eslint configs and add mocha plugin

* chore: fix mocha lint failures

* chore: remove check for cidv0 from block.put

* chore: attempt to fix hang after test

'restricts the number of concurrent connections' was hanging after running. Some things were not tied up.
@see https://github.com/ipfs/js-kubo-rpc-client/actions/runs/3642110903/jobs/6148859075

* chore: skip broken test

@see https://github.com/ipfs/js-kubo-rpc-client/tree/investigateConcurrencyTest

* fix: ipfs libs do not work with node LTS fetch

* fix: .cat beforeAll hook

* fix: .get beforeAll

* chore: update logger name

* fix: .get test 'should get a directory'

grouped files/dirs in describe

* chore: add dev helper scripts

* fix(test): should put a buffer, using defaults

* chore: remove currently broken typedocs

* fix(test): should error when removing non-existent blocks

* fix(test): should import car with roots but no blocks

* chore: add more debug output to test

* fix(test): should respect timeout option when removing a block

* chore(lint): fix lint failure

* fix(test): should import car with roots but no blocks

* fix(tests): .dag tests

* fix(test): should fail to find other peer if peer does not exist

* chore(test): disable opsPerSec expectation

* test(chore): supports concurrent writes

* fix(tests): .pin.remote.ls

* Significant DX improvement
* Significant setup improvement
* Significant speed improvement

* chore(lint): fix jsdoc complaint

* chore(test): increase .pin.remote.rm & .rmAll timeouts

* fix(test): should receive multiple messages

* fix(test): should receive messages from a different node on lots of topics

* fix(test): should unsubscribe individual handlers

* fix(test): .pubsub.peers

* chore: attempt to fix 'should list "queued" pins'

see https://github.com/ipfs/js-kubo-rpc-client/actions/runs/3662612448/jobs/6191849862

* fix(lint): unused import

* chore: attempt to clear pins prior to .pin.remote.ls 'after adding pins' group

* chore(tmp): don't fail fast in CI to ease debugging

* fix(test): .pubsub.subscribe

significant overhaul of tests. they're much more stable now

* fix(lint): build passes

* fix(test): .pin.remote.ls

* chore: cleanup .aegir and add more types

* chore: reduce duplication in js-test-and-release workflow

* chore(tmp): temporarily bail=false and fail-fast=false for non-node targets

* chore: remove tests for things not implemented in kubo

* feat: unskip should resolve IPNS link tests

* fix(test): .get directories

* chore(test): skip should default to blocking pin on chrome

* fix(lint): build succeeds

* chore(test): unskip should pin dag-cbor

* chore(test): increase 'with sharding' test timeout

see #83 (comment)

* chore: remove old go-ipfs version testing support

* fix(test): node/request-api - trailer headers

* fix: test running when not in node

* chore: move byCID function to utils

* fix(test): 'should import car with roots but no blocks' succeeds consistently

* fix(test): re-add 'should export a chunk of a file'

* fix(test): test 'should list peers this node is connected to' specifics

* chore: address the rest of the PR comments

Co-authored-by: Henrique Dias <[email protected]>

.aegir.js

@@ -1,19 +1,29 @@
 import { createServer } from 'ipfsd-ctl'
-import getPort from 'aegir/get-port'
 import EchoServer from 'aegir/echo-server'
 
+/**
+ * @typedef {object} BeforeType
+ * @property {import('ipfsd-ctl').Controller} server
+ * @property {EchoServer} echoServer
+ * @property {typeof import('./test/utils/mock-pinning-service.js')} pinningService
+ * @property {Record<string, string>} env
+ */
 /** @type {import('aegir').PartialOptions} */
 export default {
   build: {
     bundlesizeMax: '66KB'
   },
   test: {
     bail: false,
+    /**
+     *
+     * @param {Parameters<import('aegir').Options['test']['before']>[0]} options
+     * @returns {Promise<BeforeType>}
+     */
     async before (options) {
       const { PinningService } = await import('./test/utils/mock-pinning-service.js')
       const pinningService = await PinningService.start()
       const server = createServer({
-        host: '127.0.0.1',
         port: 0
       }, {
         type: 'go',
@@ -25,13 +35,18 @@ export default {
       await echoServer.start()
 
       await server.start()
+      /**
+       * @type {BeforeType}
+       */
       return {
         server,
         echoServer,
         pinningService,
         env: {
+          NODE_OPTIONS: '--no-experimental-fetch',
           IPFSD_SERVER: `http://${server.host}:${server.server.info.port}`,
           PINNING_SERVICE_ENDPOINT: pinningService.endpoint,
+          PINNING_SERVICE_KEY: 'secret',
           ECHO_SERVER: `http://${echoServer.host}:${echoServer.port}`,
         }
       }

.eslintrc.json

@@ -1,6 +1,12 @@
 
 {
+  "plugins": [
+    "jsdoc",
+    "mocha"
+  ],
   "extends": [
+    "plugin:jsdoc/recommended",
+    "plugin:mocha/recommended",
     "ipfs"
   ],
   "env": {
@@ -14,7 +20,20 @@
   "rules": {
     "no-return-await": "off",
     "no-undef-init": "off",
-    "no-warning-comments": "off"
+    "no-warning-comments": "off",
+    "mocha/max-top-level-suites": "off",
+    "mocha/no-setup-in-describe": "off"
   },
-  "ignorePatterns": [ "gists/", "vendor/", "test/fixtures/", "test/interface-tests/fixtures/"]
+  "ignorePatterns": [ "gists/", "vendor/", "test/fixtures/", "test/interface-tests/fixtures/"],
+  "overrides": [
+    {
+      "files": ["./test/interface-tests/src/**"],
+      "rules": {
+        "mocha/no-exports": "off",
+        "mocha/max-top-level-suites": "off",
+        "mocha/no-sibling-hooks": "warn",
+        "mocha/no-setup-in-describe": "off"
+      }
+    }
+  ]
 }

.github/workflows/js-test-and-release.yml

@@ -27,74 +27,35 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
-        node: [16]
-      fail-fast: true
+      fail-fast: false
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node }}
+          node-version: lts/*
       - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run --if-present test:node
       - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
         with:
           flags: node
 
-  test-chrome:
-    needs: check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: lts/*
-      - uses: ipfs/aegir/actions/cache-node-modules@master
-      - run: npm run --if-present test:chrome
-      - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
-        with:
-          flags: chrome
-
-  test-chrome-webworker:
-    needs: check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: lts/*
-      - uses: ipfs/aegir/actions/cache-node-modules@master
-      - run: npm run --if-present test:chrome-webworker
-      - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
-        with:
-          flags: chrome-webworker
-
-  test-firefox:
-    needs: check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: lts/*
-      - uses: ipfs/aegir/actions/cache-node-modules@master
-      - run: npm run --if-present test:firefox
-      - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
-        with:
-          flags: firefox
-
-  test-firefox-webworker:
+  test-non-node-target:
     needs: check
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [chrome, chrome-webworker, firefox, firefox-webworker]
+      fail-fast: false
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
           node-version: lts/*
       - uses: ipfs/aegir/actions/cache-node-modules@master
-      - run: npm run --if-present test:firefox-webworker
+      - run: npm run --if-present test:${{ matrix.target }} -- --bail=false
       - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
         with:
-          flags: firefox-webworker
+          flags: ${{ matrix.target }}
 
   test-electron-main:
     needs: check
@@ -125,7 +86,7 @@ jobs:
           flags: electron-renderer
 
   release:
-    needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-electron-main, test-electron-renderer]
+    needs: [test-node, test-non-node-target, test-electron-main, test-electron-renderer]
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && github.ref == 'refs/heads/master' # with #262 - 'refs/heads/${{{ github.default_branch }}}'
     steps:

package.json

@@ -54,18 +54,6 @@
       "require": "./dist/src/index.js"
     }
   },
-  "eslintConfig": {
-    "extends": [
-      "ipfs",
-      "plugin:jsdoc/recommended"
-    ],
-    "parserOptions": {
-      "sourceType": "module"
-    },
-    "plugins": [
-      "jsdoc"
-    ]
-  },
   "release": {
     "branches": [
       "master"
@@ -155,12 +143,15 @@
     "build": "aegir build",
     "test": "aegir test",
     "docs": "typedoc src/index.js",
-    "test:node": "aegir test -t node --cov",
+    "test:node": "aegir test -t node --cov --bail=false",
     "test:chrome": "aegir test -t browser --cov",
     "test:chrome-webworker": "aegir test -t webworker",
     "test:firefox": "aegir test -t browser -- --browser firefox",
     "test:firefox-webworker": "aegir test -t webworker -- --browser firefox",
     "lint": "aegir lint",
+    "lint:fix": "aegir lint --fix",
+    "check:push": "npm run dep-check && npm run lint && npm run build",
+    "check:push:deep": "npm run check:push && npm run test",
     "clean": "aegir clean",
     "dep-check": "aegir dep-check",
     "release": "aegir release"
@@ -188,6 +179,7 @@
   },
   "devDependencies": {
     "@ipld/car": "^5.0.0",
+    "@libp2p/interface-peer-id": "^1.0.6",
     "@libp2p/interface-peer-info": "^1.0.2",
     "@libp2p/interface-pubsub": "^3.0.0",
     "@libp2p/interfaces": "^3.0.3",
@@ -197,11 +189,12 @@
     "did-jwt": "^6.8.0",
     "eslint-config-ipfs": "^3.1.0",
     "eslint-plugin-jsdoc": "^39.3.6",
-    "go-ipfs": "^0.16.0",
+    "eslint-plugin-mocha": "^10.1.0",
+    "go-ipfs": "^0.17.0",
     "ipfs-core-types": "^0.13.0",
     "ipfs-unixfs": "^8.0.0",
     "ipfs-unixfs-importer": "^11.0.1",
-    "ipfsd-ctl": "^12.2.1",
+    "ipfsd-ctl": "^12.2.2",
     "ipns": "^4.0.0",
     "is-ipfs": "^7.0.2",
     "iso-random-stream": "^2.0.2",
@@ -222,6 +215,7 @@
     "p-defer": "^4.0.0",
     "p-map": "^5.5.0",
     "p-retry": "^5.1.1",
+    "p-timeout": "^4.1.0",
     "pako": "^2.0.4",
     "peer-id": "^0.16.0",
     "readable-stream": "^4.2.0",