Skip to content

Commit

Permalink
Port benchmarks to mitata
Browse files Browse the repository at this point in the history
* Update benchmarks to use mitata library instead of cronometro
* Move benchmarks to a separate BENCHMARKS.md file, add benchmark:update to generate it
* Fix verify benchmark: bad test PS512 token
  • Loading branch information
andolivieri-nf authored Feb 11, 2025
1 parent ffc1b8c commit ce5b923
Show file tree
Hide file tree
Showing 9 changed files with 676 additions and 262 deletions.
164 changes: 1 addition & 163 deletions README.md

Large diffs are not rendered by default.

550 changes: 550 additions & 0 deletions benchmarks/README.md

Large diffs are not rendered by default.

26 changes: 4 additions & 22 deletions benchmarks/auth0.mjs
Original file line number Diff line number Diff line change
@@ -1,32 +1,14 @@
'use strict'

import { isMainThread } from 'worker_threads'

import { tokens, privateKeys, publicKeys, compareSigning, compareVerifying, saveLogs } from './utils.mjs'

async function runSuites() {
if (!isMainThread) {
const algorightm = process.env.CURRENT_ALGORITHM

if (process.env.CURRENT_PHASE === 'sign') {
compareSigning({ a: 1, b: 2, c: 3 }, algorightm, privateKeys[algorightm], publicKeys[algorightm])
} else {
compareVerifying(tokens[algorightm], algorightm, publicKeys[algorightm])
}

return
} else {
for (const algorightm of ['HS256', 'RS256']) {
process.env.CURRENT_ALGORITHM = algorightm

process.env.CURRENT_PHASE = 'sign'
await compareSigning({ a: 1, b: 2, c: 3 }, algorightm, privateKeys[algorightm], publicKeys[algorightm])
process.env.CURRENT_PHASE = 'verify'
await compareVerifying(tokens[algorightm], algorightm, publicKeys[algorightm])
}
for (const algorithm of ['HS256', 'RS256']) {
await compareSigning({ a: 1, b: 2, c: 3 }, algorithm, privateKeys[algorithm], publicKeys[algorithm])
await compareVerifying(tokens[algorithm], algorithm, publicKeys[algorithm])
}

await saveLogs('auth0')
}

runSuites().catch(console.error)
await runSuites()
8 changes: 5 additions & 3 deletions benchmarks/decode.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ import { compareDecoding, saveLogs } from './utils.mjs'
const rsToken =
'eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJhIjoxLCJiIjoyLCJjIjozLCJpYXQiOjE1Nzk1MjEyNjF9.X04Dy1GqFCuduViZIqfKzxAXINszEEaywTwh1xU1Q2j03FCHGLe4kM6AmdMeuzPZQgcBxW-R2mGJfpcjDF83iH6fsd0Xs8Boyoml8_AEE9uZTCwpnQEXDHopmAPZ3zijbwgfrJKd0uwzMTi0iJelUhmFz65T_SlW3ZCyK150D7Xwvjq0LaieTFUbAtuJ5rpgHTtiFEtkChAb8lFl3sUYtWKPrmkcmqSQUR660j0jciLBYLG7eymsBiLJz9Knlwg6p5C_Y4hFg-oXKEEIq4G6OFdcfsBGLXhj9rogHRUDBpT_ud7SFYnpvBb3s9pgRM9y8X3eDGqVILSKGrTx4R6tpS1CGvgfUFwtFJk-wgx6JnJUcFFkrRKQ-RQK08AqPDAEZuEOictGsA7uYK5E2IpUSDiYgoZxCYx00NrwTmnvA1f_fz8vVsbfZnGLCwOmQUmFHl3MLZTLk9ti0dW5dWwZU4u-4qTvvytLF4jEKEfvnCv6IjnfYfBo9nAh6zTW5lueT3rehre0lhW6wxfjgflTafeq2C8PV89t1vvy-iIxTz5PoXN-GeyEdChtjbzfT0Tg2pdMAmT6fisGKIYioqSG_0ugn7SskgYrH_SSk8UzkJFd0ksG5DJ3YYwjmrRi3Ll8S46DoxX6v7NOsq9xPiV4wTc8yQnK7zG2P5MdX-uJBK8'

async function runSuites() {
await compareDecoding(rsToken, 'RS512')
export async function runSuites() {
const result = await compareDecoding(rsToken, 'RS512')

await saveLogs('decode')

return [{ algorithm: 'RS512', result }]
}

runSuites().catch(console.error)
if (import.meta.filename === process.argv[1]) await runSuites()
19 changes: 7 additions & 12 deletions benchmarks/sign.mjs
Original file line number Diff line number Diff line change
@@ -1,21 +1,16 @@
'use strict'

import { isMainThread } from 'worker_threads'
import { privateKeys, publicKeys, compareSigning, saveLogs } from './utils.mjs'

async function runSuites() {
if (!isMainThread) {
const algorightm = process.env.CURRENT_ALGORITHM
compareSigning({ a: 1, b: 2, c: 3 }, algorightm, privateKeys[algorightm], publicKeys[algorightm])
return
} else {
for (const algorightm of ['HS512', 'ES512', 'RS512', 'PS512', 'EdDSA']) {
process.env.CURRENT_ALGORITHM = algorightm
await compareSigning({ a: 1, b: 2, c: 3 }, algorightm, privateKeys[algorightm], publicKeys[algorightm])
}
export async function runSuites() {
const benchmarkResults = []
for (const algorithm of ['HS512', 'ES512', 'RS512', 'PS512', 'EdDSA']) {
const result = await compareSigning({ a: 1, b: 2, c: 3 }, algorithm, privateKeys[algorithm], publicKeys[algorithm])
benchmarkResults.push({ algorithm, result })
}

await saveLogs('sign')
return benchmarkResults
}

runSuites().catch(console.error)
if (import.meta.filename === process.argv[1]) await runSuites()
42 changes: 42 additions & 0 deletions benchmarks/update_benchmarks.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import { runSuites as runSignSuites } from './sign.mjs'
import { runSuites as runVerifySuites } from './verify.mjs'
import { runSuites as runDecodeSuites } from './decode.mjs'
import { writeFile } from 'fs/promises'
import { join } from 'path'

const signBenchmark = await runSignSuites().catch(console.error)
const decodeBenchmark = await runDecodeSuites().catch(console.error)
const verifyBenchmark = await runVerifySuites().catch(console.error)

const printDetail = ({ algorithm, result }) =>
`
<details>
<summary>${algorithm}</summary>
## ${algorithm}
\`\`\`
${result}
\`\`\`
</details>
`

const pageMarkdownContent = `# Benchmarks
Made with [mitata](https://github.com/evanwashere/mitata) library
## Signing
${signBenchmark.map(printDetail).join('\n')}
## Decoding
${decodeBenchmark.map(printDetail).join('\n')}
Note that for decoding the algorithm is irrelevant, so only one was measured.
## Verifying
${verifyBenchmark.map(printDetail).join('\n')}
`

await writeFile(join(import.meta.dirname, 'README.md'), pageMarkdownContent, 'utf8')
105 changes: 57 additions & 48 deletions benchmarks/utils.mjs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict'

import nodeRsJwt, { Algorithm } from '@node-rs/jsonwebtoken'
import cronometro from 'cronometro'
import { run, bench, summary } from 'mitata'
import { readFileSync } from 'fs'
import { mkdir, writeFile } from 'fs/promises'
import { dirname, resolve } from 'path'
Expand All @@ -22,13 +22,11 @@ const { sign: nodeRsSign, signSync: nodeRsSignSync, verifySync: nodeRsVerifySync
const { sign: joseSign, verify: joseVerify, decode: joseDecode } = JWTJose
const { asKey } = JWKJose

const iterations = process.env.BENCHMARK_ITERATIONS || 10000

const output = []
const cronometroOptions = {
iterations: Number.parseInt(iterations, 10),
warmup: true,
print: { compare: true, compareMode: 'base' }
const mitataOptions = {
format: 'mitata',
colors: false,
throw: true
}

export const tokens = {
Expand All @@ -47,7 +45,7 @@ export const tokens = {
RS512:
'eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCIsImtpZCI6IjEyMyJ9.eyJhIjoxLCJiIjoyLCJjIjozLCJpYXQiOjE1ODA5MTUyOTZ9.JEHZDpA99hww-5-PKcCvwialNy1QcyDSJXJ-qvzV0fU56NXPxZRfn2rEwdX4g-8N-oXLsjCjNZzr0Hl39FXSK0ke_vnzwPW6D4r5mVL6Ak0K-jMgNFidxElM7PRg2XE7N72dI5ClQJCJMqex7NYmIN4OUj9psx1NDv8bM_Oj44kXyI6ozYrkV-6tvowLXUX9BOZH55jF3aAA1DLI4rVBKc_JYqiHf376xu6zvFxzZ8XP3-S-dTR7OBRZLe5_Y6YJweWiL2n0lRkEjYrpK3Ht9MlfaCmW2_KMH0DpUKVS6nnKmzqGjdutnzP6PYXZsJikCQOrIcPW97LdQWLLRIptSpn7YHH1xbNbq__kryaggwpKuNd6qhdXqREEhpaYl3Xc4yjGnBR0zMq7J-GxDo7mSujMMFmb4ZQLQWwANCEHSfqYIJYp7Upc1Rd__lo56Mr1Bd9claZPBNgKqAvhlmjZT9lELA-eyEzhH_yrcVzMcVpCC_oVIzvpiDQ0jgOLcDIz0q8uzoSCks3M0IK3flefopY8g1e-OExqBoYrfoktFciabLfTM5g-rIpC0CrrDN-TfXLqZAPkIv7suGBmQn9-HbcFL8eZEIg-q6D3o7EAfCO7ki9ncrm47C2y5SX3zDOjG37_5pN2JGWztfSFRQ-YbbEV13-TuKvRG3HLJjJQk6g',
PS512:
'eyJhbGciOiJQUzUxMiIsInR5cCI6IkpXVCIsImtpZCI6IjEyMyJ9.eyJhIjoxLCJiIjoyLCJjIjozLCJpYXQiOjE1ODA5MTUyOTd9.IfPgE2aPz9dBzvb8hvn_RjBvdq5jLgfzRw-lM5Q2Ah67NYuRYjCzpvywJASY-0Y-tvk94kwwmDUpqR7nPPPlcv9o_OYQVGhPnndh6iMww0D-MGZwP0sqIauu6NgUsCY4rFG2_K8lxCYbdNThJJVDDN8v_VmKrK3qh7DJC89PE-ZbIMr4N3AuLww-vPgB-9hFmuVnjgO43scZb8C_SaA1HuSbw_SU6OWguAlaTP3zUKlyQmTvvx843J8byi5jDcqK4Rtah5gRaO8U1l6bzmVCKs8Fh3tv7A7GWs-eFukFu5dUr-Ig0iyIhPSAVOjnk0dEZ4s5YI1XaPrnm3wAKV9fyzSri2LaaElp_8Cy7xyJNDPgWSTUmm4BGU7m5x_zwamRbQ1zI7p-YxftwkL4Jl8VD1km7CP9T_6cOEt6RzSTNbTgkk3XhcqYZ0oTgAJ4nXa4j47-7E0n5drtM1xoYeWBaWQvXPdMwGTAwXMx33B1WOm80B7Ncn6AzZKtJtEYalFEKntNfJhWi5x9nZNc4-3cja4o1appVm5PWSOtV4mkLsrLL1T0x1c9ymyF_XtYkRAuxdOKaRs2N5YxHcikgX-iifI1Ih4l79BrWAgyioGjMTU78VMV_gLRQ1VLexmgYJLKL2fBUrBR-k8fI4VwbKEtEZF3wBINWBAp4-urFV3QVig',
'eyJhbGciOiJQUzUxMiIsInR5cCI6IkpXVCJ9.eyJhIjoxLCJiIjoyLCJjIjozfQ.iUpS__v2Lcw-1ahbVrnoLzi0SeW54YCkn0LMlsPdpmgUR09KR0MTYh949oQK0OpiMj0ZyHBjynn8v8-pbY0Y9g2Q-fT0zX18s55trHxv1ynBXmDQjiA_RNHWBgWfxRFYu9eqIE6MQcbQon7dTPptW7irPkUNS3inBesnRGx9k9hrRlMqp5ncB9XENJYDtj20oIIutCXzmUM3m4vrrVdIlwpTCb_bs6TK6oYtJC9TjPwX6UwCMp-0vEOQdmgf0qhW4LkgGzg0IRhJ-gc6ETVqOuKKTKcV2DYc9vy0sNh4gMRX-1N3y63Q_nNdSdtkCYy6yrzIXpoV-nhtxgPezsapdTRrqLD_3cITvq8YN1EHxZ8UF1ps7TB2sMIQOP4bH5a7pGzLhy_HjKp1kORkqVthXVaEQ3Q1p9FC7uligGCg6GWgDCYi4wTWJCSH00qWt9NG_8HMSHBWRAJf8zyEplJlY6WAfCG-qYQCqGYd8-yG1exMBEu7vuyumW-41gRxa1q5rT_2oI8N1WCqn-pPzXfEQ8NpBpzZucfUtSFp7T2V_a_86r4-tcQXvX0N5iGRs3KT-60AMdDDO2GePrBubkq4DFniSgpX-1-DNdJbwZf0Ip94ogU5Np9i2syoRR8mtGDXKPG_My4xA6gk8WqM-GoVHtpjhuclZ-EJCjRKVqWiRz8',
EdDSA:
'eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCIsImt0eSI6Ik9LUCIsImNydiI6IkVkMjU1MTkiLCJraWQiOiIxMjMifQ.eyJhIjoxLCJiIjoyLCJjIjozLCJpYXQiOjE1ODY3ODQ3ODF9.PDIxWOWhAHr-7Zy7UC8W4pRuk7dMYTD8xy0DR0N102P0pXK6U4r6THHe66muTdSM3qiDHZnync1WQp-10QFLCQ'
}
Expand Down Expand Up @@ -104,29 +102,28 @@ export function compareDecoding(token, algorithm) {
log('-------')
}

return cronometro(
{
[`${algorithm} - fast-jwt`]: function () {
fastjwtDecoder(token)
},
[`${algorithm} - fast-jwt (complete)`]: function () {
fastjwtCompleteDecoder(token)
},
[`${algorithm} - jsonwebtoken`]: function () {
jsonwebtokenDecode(token)
},
[`${algorithm} - jsonwebtoken (complete)`]: function () {
jsonwebtokenDecode(token, { complete: true })
},
[`${algorithm} - jose`]: function () {
joseDecode(token)
},
[`${algorithm} - jose (complete)`]: function () {
joseDecode(token, { complete: true })
}
const tests = {
[`${algorithm} - fast-jwt`]: function () {
fastjwtDecoder(token)
},
[`${algorithm} - fast-jwt (complete)`]: function () {
fastjwtCompleteDecoder(token)
},
[`${algorithm} - jsonwebtoken`]: function () {
jsonwebtokenDecode(token)
},
cronometroOptions
)
[`${algorithm} - jsonwebtoken (complete)`]: function () {
jsonwebtokenDecode(token, { complete: true })
},
[`${algorithm} - jose`]: function () {
joseDecode(token)
},
[`${algorithm} - jose (complete)`]: function () {
joseDecode(token, { complete: true })
}
}

return runMitata(tests, mitataOptions)
}

export async function compareSigning(payload, algorithm, privateKey, publicKey) {
Expand Down Expand Up @@ -186,8 +183,8 @@ export async function compareSigning(payload, algorithm, privateKey, publicKey)
[`${algorithm} - jsonwebtoken (sync)`]: function () {
jsonwebtokenSign(payload, privateKey, { algorithm, noTimestamp: true })
},
[`${algorithm} - jsonwebtoken (async)`]: function (done) {
jsonwebtokenSign(payload, privateKey, { algorithm, noTimestamp: true }, done)
[`${algorithm} - jsonwebtoken (async)`]: async function () {
return new Promise(resolve => jsonwebtokenSign(payload, privateKey, { algorithm, noTimestamp: true }, resolve))
}
})
}
Expand All @@ -196,21 +193,18 @@ export async function compareSigning(payload, algorithm, privateKey, publicKey)
[`${algorithm} - fast-jwt (sync)`]: function () {
fastjwtSign(payload)
},
[`${algorithm} - fast-jwt (async)`]: function (done) {
fastjwtSignAsync(payload, done)
}
})

Object.assign(tests, {
[`${algorithm} - fast-jwt (async)`]: async function () {
return fastjwtSignAsync(payload)
},
[`${algorithm} - @node-rs/jsonwebtoken (sync)`]: function () {
nodeRsSignSync({ data: payload }, privateKey, { algorithm: Algorithm[algorithm.toUpperCase()] })
},
[`${algorithm} - @node-rs/jsonwebtoken (async)`]: function (done) {
nodeRsSign({ data: payload }, privateKey, { algorithm: Algorithm[algorithm.toUpperCase()] }).then(() => done())
[`${algorithm} - @node-rs/jsonwebtoken (async)`]: async function () {
return nodeRsSign({ data: payload }, privateKey, { algorithm: Algorithm[algorithm.toUpperCase()] })
}
})

return cronometro(tests, cronometroOptions)
return runMitata(tests, mitataOptions)
}

export function compareVerifying(token, algorithm, publicKey) {
Expand Down Expand Up @@ -240,14 +234,14 @@ export function compareVerifying(token, algorithm, publicKey) {
[`${algorithm} - fast-jwt (sync)`]: function () {
fastjwtVerify(token)
},
[`${algorithm} - fast-jwt (async)`]: function (done) {
fastjwtVerifyAsync(token, done)
[`${algorithm} - fast-jwt (async)`]: async function () {
return fastjwtVerifyAsync(token)
},
[`${algorithm} - fast-jwt (sync with cache)`]: function () {
fastjwtCachedVerify(token)
},
[`${algorithm} - fast-jwt (async with cache)`]: function (done) {
fastjwtCachedVerifyAsync(token, done)
[`${algorithm} - fast-jwt (async with cache)`]: async function () {
return fastjwtCachedVerifyAsync(token)
},
[`${algorithm} - jose (sync)`]: function () {
joseVerify(token, josePublicKey)
Expand All @@ -258,10 +252,25 @@ export function compareVerifying(token, algorithm, publicKey) {
tests[`${algorithm} - jsonwebtoken (sync)`] = function () {
jsonwebtokenVerify(token, publicKey)
}
tests[`${algorithm} - jsonwebtoken (async)`] = function (done) {
jsonwebtokenVerify(token, publicKey, done)
tests[`${algorithm} - jsonwebtoken (async)`] = async function () {
return new Promise(resolve => jsonwebtokenVerify(token, publicKey, resolve))
}
}

return cronometro(tests, cronometroOptions)
return runMitata(tests, mitataOptions)
}

async function runMitata(tests, opts) {
const outputLines = []
opts.print = line => {
console.log(line)
outputLines.push(line)
}

summary(() => {
Object.entries(tests).forEach(([t, fn]) => bench(t, fn))
})

await run(opts)
return outputLines.join('\n')
}
19 changes: 7 additions & 12 deletions benchmarks/verify.mjs
Original file line number Diff line number Diff line change
@@ -1,21 +1,16 @@
'use strict'

import { isMainThread } from 'worker_threads'
import { tokens, publicKeys, compareVerifying, saveLogs } from './utils.mjs'

async function runSuites() {
if (!isMainThread) {
const algorightm = process.env.CURRENT_ALGORITHM
compareVerifying(tokens[algorightm], algorightm, publicKeys[algorightm])
return
} else {
for (const algorightm of ['HS512', 'ES512', 'RS512', 'PS512', 'EdDSA']) {
process.env.CURRENT_ALGORITHM = algorightm
await compareVerifying(tokens[algorightm], algorightm, publicKeys[algorightm])
}
export async function runSuites() {
const benchmarkResults = []
for (const algorithm of ['HS512', 'ES512', 'RS512', 'PS512', 'EdDSA']) {
const result = await compareVerifying(tokens[algorithm], algorithm, publicKeys[algorithm])
benchmarkResults.push({ algorithm, result })
}

await saveLogs('verify')
return benchmarkResults
}

runSuites().catch(console.error)
if (import.meta.filename === process.argv[1]) await runSuites()
5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@
"benchmark:sign": "node benchmarks/sign.mjs",
"benchmark:decode": "node benchmarks/decode.mjs",
"benchmark:verify": "node benchmarks/verify.mjs",
"benchmark:auth0": "node benchmarks/auth0.mjs"
"benchmark:auth0": "node benchmarks/auth0.mjs",
"benchmark:update": "node benchmarks/update_benchmarks.mjs"
},
"dependencies": {
"@lukeed/ms": "^2.0.2",
Expand All @@ -59,7 +60,6 @@
"devDependencies": {
"@node-rs/jsonwebtoken": "^0.5.9",
"@types/node": "^22.10.2",
"cronometro": "^4.0.0",
"eslint": "^9.17.0",
"eslint-config-prettier": "^10.0.1",
"eslint-plugin-import": "^2.31.0",
Expand All @@ -69,6 +69,7 @@
"fastify": "^5.2.0",
"jose": "^2.0.7",
"jsonwebtoken": "^9.0.2",
"mitata": "^1.0.34",
"prettier": "^3.4.2",
"tsd": "^0.31.2",
"typescript": "^5.7.2",
Expand Down

0 comments on commit ce5b923

Please sign in to comment.