diff --git a/packages/enr/src/relay_shard_codec.spec.ts b/packages/enr/src/relay_shard_codec.spec.ts index 38857dba49..985f61b35c 100644 --- a/packages/enr/src/relay_shard_codec.spec.ts +++ b/packages/enr/src/relay_shard_codec.spec.ts @@ -6,7 +6,7 @@ import { decodeRelayShard, encodeRelayShard } from "./relay_shard_codec.js"; describe("Relay Shard codec", () => { // Boundary test case it("should handle a minimal index list", () => { - const shardInfo = { cluster: 0, indexList: [0] }; + const shardInfo = { clusterId: 0, shards: [0] }; const encoded = encodeRelayShard(shardInfo); const decoded = decodeRelayShard(encoded); expect(decoded).to.deep.equal( @@ -23,8 +23,8 @@ describe("Relay Shard codec", () => { fc .array(fc.nat(1023), { minLength: 1, maxLength: 63 }) // indexList .map((arr) => [...new Set(arr)].sort((a, b) => a - b)), - (cluster, indexList) => { - const shardInfo = { cluster, indexList }; + (clusterId, shards) => { + const shardInfo = { clusterId, shards }; const encoded = encodeRelayShard(shardInfo); const decoded = decodeRelayShard(encoded); @@ -45,8 +45,8 @@ describe("Relay Shard codec", () => { fc .array(fc.nat(1023), { minLength: 64, maxLength: 1024 }) // indexList .map((arr) => [...new Set(arr)].sort((a, b) => a - b)), - (cluster, indexList) => { - const shardInfo = { cluster, indexList }; + (clusterId, shards) => { + const shardInfo = { clusterId, shards }; const encoded = encodeRelayShard(shardInfo); const decoded = decodeRelayShard(encoded); diff --git a/packages/enr/src/relay_shard_codec.ts b/packages/enr/src/relay_shard_codec.ts index 9eaad8fffd..91dea7b4ea 100644 --- a/packages/enr/src/relay_shard_codec.ts +++ b/packages/enr/src/relay_shard_codec.ts @@ -8,9 +8,9 @@ export const decodeRelayShard = (bytes: Uint8Array): ShardInfo => { if (bytes.length < 3) throw new Error("Insufficient data"); const view = new DataView(bytes.buffer); - const cluster = view.getUint16(0); + const clusterId = view.getUint16(0); - const indexList = []; + const shards = []; if (bytes.length === 130) { // rsv format (Bit Vector) @@ -18,7 +18,7 @@ export const decodeRelayShard = (bytes: Uint8Array): ShardInfo => { const byteIndex = Math.floor(i / 8) + 2; // Adjusted for the 2-byte cluster field const bitIndex = 7 - (i % 8); if (view.getUint8(byteIndex) & (1 << bitIndex)) { - indexList.push(i); + shards.push(i); } } } else { @@ -26,33 +26,33 @@ export const decodeRelayShard = (bytes: Uint8Array): ShardInfo => { const numIndices = view.getUint8(2); for (let i = 0, offset = 3; i < numIndices; i++, offset += 2) { if (offset + 1 >= bytes.length) throw new Error("Unexpected end of data"); - indexList.push(view.getUint16(offset)); + shards.push(view.getUint16(offset)); } } - return { cluster, indexList }; + return { clusterId, shards }; }; export const encodeRelayShard = (shardInfo: ShardInfo): Uint8Array => { - const { cluster, indexList } = shardInfo; - const totalLength = indexList.length >= 64 ? 130 : 3 + 2 * indexList.length; + const { clusterId, shards } = shardInfo; + const totalLength = shards.length >= 64 ? 130 : 3 + 2 * shards.length; const buffer = new ArrayBuffer(totalLength); const view = new DataView(buffer); - view.setUint16(0, cluster); + view.setUint16(0, clusterId); - if (indexList.length >= 64) { + if (shards.length >= 64) { // rsv format (Bit Vector) - for (const index of indexList) { + for (const index of shards) { const byteIndex = Math.floor(index / 8) + 2; // Adjusted for the 2-byte cluster field const bitIndex = 7 - (index % 8); view.setUint8(byteIndex, view.getUint8(byteIndex) | (1 << bitIndex)); } } else { // rs format (Index List) - view.setUint8(2, indexList.length); - for (let i = 0, offset = 3; i < indexList.length; i++, offset += 2) { - view.setUint16(offset, indexList[i]); + view.setUint8(2, shards.length); + for (let i = 0, offset = 3; i < shards.length; i++, offset += 2) { + view.setUint16(offset, shards[i]); } } diff --git a/packages/interfaces/src/enr.ts b/packages/interfaces/src/enr.ts index b06d51cacd..2aa5e44487 100644 --- a/packages/interfaces/src/enr.ts +++ b/packages/interfaces/src/enr.ts @@ -19,8 +19,8 @@ export interface Waku2 { } export interface ShardInfo { - cluster: number; - indexList: number[]; + clusterId: number; + shards: number[]; } export interface IEnr extends Map { diff --git a/packages/interfaces/src/message.ts b/packages/interfaces/src/message.ts index 137225cd58..40de174026 100644 --- a/packages/interfaces/src/message.ts +++ b/packages/interfaces/src/message.ts @@ -1,8 +1,8 @@ import type { PubsubTopic } from "./misc.js"; export interface SingleShardInfo { - cluster: number; - index: number; + clusterId: number; + shard: number; } export interface IRateLimitProof { diff --git a/packages/tests/src/node/interfaces.ts b/packages/tests/src/node/interfaces.ts index 3a19a85134..f805576268 100644 --- a/packages/tests/src/node/interfaces.ts +++ b/packages/tests/src/node/interfaces.ts @@ -24,6 +24,7 @@ export interface Args { discv5UdpPort?: number; // `legacyFilter` is required to enable filter v1 with go-waku legacyFilter?: boolean; + clusterId?: number; } export enum LogLevel { diff --git a/packages/tests/tests/filter/multiple_pubsub.node.spec.ts b/packages/tests/tests/filter/multiple_pubsub.node.spec.ts index a8401fadeb..4be4cac9d3 100644 --- a/packages/tests/tests/filter/multiple_pubsub.node.spec.ts +++ b/packages/tests/tests/filter/multiple_pubsub.node.spec.ts @@ -32,16 +32,16 @@ describe("Waku Filter V2: Multiple PubsubTopics", function () { let messageCollector: MessageCollector; const customPubsubTopic1 = singleShardInfoToPubsubTopic({ - cluster: 3, - index: 1 + clusterId: 3, + shard: 1 }); const customPubsubTopic2 = singleShardInfoToPubsubTopic({ - cluster: 3, - index: 2 + clusterId: 3, + shard: 2 }); - const shardInfo: ShardInfo = { cluster: 3, indexList: [1, 2] }; - const singleShardInfo1: SingleShardInfo = { cluster: 3, index: 1 }; - const singleShardInfo2: SingleShardInfo = { cluster: 3, index: 2 }; + const shardInfo: ShardInfo = { clusterId: 3, shards: [1, 2] }; + const singleShardInfo1: SingleShardInfo = { clusterId: 3, shard: 1 }; + const singleShardInfo2: SingleShardInfo = { clusterId: 3, shard: 2 }; const customContentTopic1 = "/test/2/waku-filter"; const customContentTopic2 = "/test/3/waku-filter"; const customEncoder1 = createEncoder({ diff --git a/packages/tests/tests/light-push/multiple_pubsub.node.spec.ts b/packages/tests/tests/light-push/multiple_pubsub.node.spec.ts index 5017239c4a..c598122ab4 100644 --- a/packages/tests/tests/light-push/multiple_pubsub.node.spec.ts +++ b/packages/tests/tests/light-push/multiple_pubsub.node.spec.ts @@ -27,16 +27,16 @@ describe("Waku Light Push : Multiple PubsubTopics", function () { let nwaku2: NimGoNode; let messageCollector: MessageCollector; const customPubsubTopic1 = singleShardInfoToPubsubTopic({ - cluster: 3, - index: 1 + clusterId: 3, + shard: 1 }); const customPubsubTopic2 = singleShardInfoToPubsubTopic({ - cluster: 3, - index: 2 + clusterId: 3, + shard: 2 }); - const shardInfo: ShardInfo = { cluster: 3, indexList: [1, 2] }; - const singleShardInfo1: SingleShardInfo = { cluster: 3, index: 1 }; - const singleShardInfo2: SingleShardInfo = { cluster: 3, index: 2 }; + const shardInfo: ShardInfo = { clusterId: 3, shards: [1, 2] }; + const singleShardInfo1: SingleShardInfo = { clusterId: 3, shard: 1 }; + const singleShardInfo2: SingleShardInfo = { clusterId: 3, shard: 2 }; const customContentTopic1 = "/test/2/waku-light-push/utf8"; const customContentTopic2 = "/test/3/waku-light-push/utf8"; const customEncoder1 = createEncoder({ diff --git a/packages/tests/tests/relay/multiple_pubsub.node.spec.ts b/packages/tests/tests/relay/multiple_pubsub.node.spec.ts index 97bffe0cb9..a870f0cba2 100644 --- a/packages/tests/tests/relay/multiple_pubsub.node.spec.ts +++ b/packages/tests/tests/relay/multiple_pubsub.node.spec.ts @@ -27,24 +27,24 @@ describe("Waku Relay, multiple pubsub topics", function () { let waku3: RelayNode; const customPubsubTopic1 = singleShardInfoToPubsubTopic({ - cluster: 3, - index: 1 + clusterId: 3, + shard: 1 }); const customPubsubTopic2 = singleShardInfoToPubsubTopic({ - cluster: 3, - index: 2 + clusterId: 3, + shard: 2 }); - const shardInfo1: ShardInfo = { cluster: 3, indexList: [1] }; + const shardInfo1: ShardInfo = { clusterId: 3, shards: [1] }; const singleShardInfo1: SingleShardInfo = { - cluster: 3, - index: 1 + clusterId: 3, + shard: 1 }; const customContentTopic1 = "/test/2/waku-relay/utf8"; const customContentTopic2 = "/test/3/waku-relay/utf8"; - const shardInfo2: ShardInfo = { cluster: 3, indexList: [2] }; + const shardInfo2: ShardInfo = { clusterId: 3, shards: [2] }; const singleShardInfo2: SingleShardInfo = { - cluster: 3, - index: 2 + clusterId: 3, + shard: 2 }; const customEncoder1 = createEncoder({ pubsubTopicShardInfo: singleShardInfo1, @@ -56,7 +56,7 @@ describe("Waku Relay, multiple pubsub topics", function () { contentTopic: customContentTopic2 }); const customDecoder2 = createDecoder(customContentTopic2, singleShardInfo2); - const shardInfoBothShards: ShardInfo = { cluster: 3, indexList: [1, 2] }; + const shardInfoBothShards: ShardInfo = { clusterId: 3, shards: [1, 2] }; afterEach(async function () { this.timeout(15000); diff --git a/packages/tests/tests/relay/publish.node.spec.ts b/packages/tests/tests/relay/publish.node.spec.ts index 5a243397f6..f550bb4a9e 100644 --- a/packages/tests/tests/relay/publish.node.spec.ts +++ b/packages/tests/tests/relay/publish.node.spec.ts @@ -128,7 +128,7 @@ describe("Waku Relay, Publish", function () { it("Fails to publish message with wrong pubsubtopic", async function () { const wrong_encoder = createEncoder({ - pubsubTopicShardInfo: { cluster: 3, index: 1 }, + pubsubTopicShardInfo: { clusterId: 3, shard: 1 }, contentTopic: TestContentTopic }); const pushResponse = await waku1.relay.send(wrong_encoder, { diff --git a/packages/tests/tests/sharding/peer_management.spec.ts b/packages/tests/tests/sharding/peer_management.spec.ts index 13fe53d68d..833795eb18 100644 --- a/packages/tests/tests/sharding/peer_management.spec.ts +++ b/packages/tests/tests/sharding/peer_management.spec.ts @@ -40,9 +40,9 @@ describe("Static Sharding: Peer Management", function () { this.timeout(100_000); const pubsubTopics = [ - singleShardInfoToPubsubTopic({ cluster: 18, index: 2 }) + singleShardInfoToPubsubTopic({ clusterId: 18, shard: 2 }) ]; - const shardInfo: ShardInfo = { cluster: 18, indexList: [2] }; + const shardInfo: ShardInfo = { clusterId: 18, shards: [2] }; await nwaku1.start({ pubsubTopic: pubsubTopics, @@ -112,11 +112,11 @@ describe("Static Sharding: Peer Management", function () { it("px service nodes not subscribed to the shard should not be dialed", async function () { this.timeout(100_000); const pubsubTopicsToDial = [ - singleShardInfoToPubsubTopic({ cluster: 18, index: 2 }) + singleShardInfoToPubsubTopic({ clusterId: 18, shard: 2 }) ]; - const shardInfoToDial: ShardInfo = { cluster: 18, indexList: [2] }; + const shardInfoToDial: ShardInfo = { clusterId: 18, shards: [2] }; const pubsubTopicsToIgnore = [ - singleShardInfoToPubsubTopic({ cluster: 18, index: 1 }) + singleShardInfoToPubsubTopic({ clusterId: 18, shard: 1 }) ]; // this service node is not subscribed to the shard diff --git a/packages/tests/tests/sharding/running_nodes.spec.ts b/packages/tests/tests/sharding/running_nodes.spec.ts index 6fedb6da02..043c0db423 100644 --- a/packages/tests/tests/sharding/running_nodes.spec.ts +++ b/packages/tests/tests/sharding/running_nodes.spec.ts @@ -8,17 +8,17 @@ import { makeLogFileName } from "../../src/log_file.js"; import { NimGoNode } from "../../src/node/node.js"; const PubsubTopic1 = singleShardInfoToPubsubTopic({ - cluster: 0, - index: 2 + clusterId: 0, + shard: 2 }); const PubsubTopic2 = singleShardInfoToPubsubTopic({ - cluster: 0, - index: 3 + clusterId: 0, + shard: 3 }); -const shardInfoFirstShard: ShardInfo = { cluster: 0, indexList: [2] }; -const shardInfoBothShards: ShardInfo = { cluster: 0, indexList: [2, 3] }; -const singleShardInfo1: SingleShardInfo = { cluster: 0, index: 2 }; -const singleShardInfo2: SingleShardInfo = { cluster: 0, index: 3 }; +const shardInfoFirstShard: ShardInfo = { clusterId: 0, shards: [2] }; +const shardInfoBothShards: ShardInfo = { clusterId: 0, shards: [2, 3] }; +const singleShardInfo1: SingleShardInfo = { clusterId: 0, shard: 2 }; +const singleShardInfo2: SingleShardInfo = { clusterId: 0, shard: 3 }; const ContentTopic = "/waku/2/content/test.js"; describe("Static Sharding: Running Nodes", () => { diff --git a/packages/tests/tests/store/utils.ts b/packages/tests/tests/store/utils.ts index 1bc5ba4fe9..4a52290f66 100644 --- a/packages/tests/tests/store/utils.ts +++ b/packages/tests/tests/store/utils.ts @@ -19,25 +19,25 @@ export const TestContentTopic = "/test/1/waku-store/utf8"; export const TestEncoder = createEncoder({ contentTopic: TestContentTopic }); export const TestDecoder = createDecoder(TestContentTopic); export const customShardedPubsubTopic1 = singleShardInfoToPubsubTopic({ - cluster: 3, - index: 1 + clusterId: 3, + shard: 1 }); export const customShardedPubsubTopic2 = singleShardInfoToPubsubTopic({ - cluster: 3, - index: 2 + clusterId: 3, + shard: 2 }); -export const shardInfo1: ShardInfo = { cluster: 3, indexList: [1] }; +export const shardInfo1: ShardInfo = { clusterId: 3, shards: [1] }; export const customContentTopic1 = "/test/2/waku-store/utf8"; export const customContentTopic2 = "/test/3/waku-store/utf8"; export const customDecoder1 = createDecoder(customContentTopic1, { - cluster: 3, - index: 1 + clusterId: 3, + shard: 1 }); export const customDecoder2 = createDecoder(customContentTopic2, { - cluster: 3, - index: 2 + clusterId: 3, + shard: 2 }); -export const shardInfoBothShards: ShardInfo = { cluster: 3, indexList: [1, 2] }; +export const shardInfoBothShards: ShardInfo = { clusterId: 3, shards: [1, 2] }; export const totalMsgs = 20; export const messageText = "Store Push works!"; diff --git a/packages/utils/src/common/sharding.ts b/packages/utils/src/common/sharding.ts index afaecb2b67..648cb72b17 100644 --- a/packages/utils/src/common/sharding.ts +++ b/packages/utils/src/common/sharding.ts @@ -6,20 +6,20 @@ import { concat, utf8ToBytes } from "../bytes/index.js"; export const singleShardInfoToPubsubTopic = ( shardInfo: SingleShardInfo ): PubsubTopic => { - if (shardInfo.cluster === undefined || shardInfo.index === undefined) + if (shardInfo.clusterId === undefined || shardInfo.shard === undefined) throw new Error("Invalid shard"); - return `/waku/2/rs/${shardInfo.cluster}/${shardInfo.index}`; + return `/waku/2/rs/${shardInfo.clusterId}/${shardInfo.shard}`; }; export const shardInfoToPubsubTopics = ( shardInfo: ShardInfo ): PubsubTopic[] => { - if (shardInfo.cluster === undefined || shardInfo.indexList === undefined) + if (shardInfo.clusterId === undefined || shardInfo.shards === undefined) throw new Error("Invalid shard"); - return shardInfo.indexList.map( - (index) => `/waku/2/rs/${shardInfo.cluster}/${index}` + return shardInfo.shards.map( + (index) => `/waku/2/rs/${shardInfo.clusterId}/${index}` ); }; @@ -27,13 +27,25 @@ export const pubsubTopicToSingleShardInfo = ( pubsubTopics: PubsubTopic ): SingleShardInfo => { const parts = pubsubTopics.split("/"); - if (parts.length != 6) throw new Error("Invalid pubsub topic"); - const cluster = parseInt(parts[4]); - const index = parseInt(parts[5]); - if (isNaN(cluster) || isNaN(index)) throw new Error("Invalid pubsub topic"); + if ( + parts.length != 6 || + parts[1] !== "waku" || + parts[2] !== "2" || + parts[3] !== "rs" + ) + throw new Error("Invalid pubsub topic"); - return { cluster, index }; + const clusterId = parseInt(parts[4]); + const shard = parseInt(parts[5]); + + if (isNaN(clusterId) || isNaN(shard)) + throw new Error("Invalid clusterId or shard"); + + return { + clusterId, + shard + }; }; export function ensurePubsubTopicIsConfigured(