From dda12abe6ded412254e63d199b0fb3375e182187 Mon Sep 17 00:00:00 2001 From: "Dr. Holomorfo" Date: Wed, 11 Dec 2024 23:13:26 -0800 Subject: [PATCH 1/3] benchmark render config and sample report --- package.json | 2 + test/bench/REPORTS/REPORT_1_TEST CPU.md | 134 ++++++++++++++++ test/bench/cpu_transforms.bench.js | 168 ++++++++++++++++++++ test/bench/poc.bench_.js | 29 ++++ test/bench/rendering.bench.js | 199 ++++++++++++++++++++++++ vitest.workspace.mjs | 3 +- 6 files changed, 534 insertions(+), 1 deletion(-) create mode 100644 test/bench/REPORTS/REPORT_1_TEST CPU.md create mode 100644 test/bench/cpu_transforms.bench.js create mode 100644 test/bench/poc.bench_.js create mode 100644 test/bench/rendering.bench.js diff --git a/package.json b/package.json index e34e3facd0..cd2ffd5de8 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,8 @@ "dev": "vite preview/", "dev:global": "concurrently -n build,server \"rollup -c -w\" \"npx vite preview/global/\"", "docs": "documentation build ./src/**/*.js ./src/**/**/*.js -o ./docs/data.json && node ./utils/convert.js", + "bench": "vitest bench", + "bench:report": "vitest bench --reporter=verbose", "test": "vitest", "lint": "eslint .", "lint:fix": "eslint --fix ." diff --git a/test/bench/REPORTS/REPORT_1_TEST CPU.md b/test/bench/REPORTS/REPORT_1_TEST CPU.md new file mode 100644 index 0000000000..f645c2f235 --- /dev/null +++ b/test/bench/REPORTS/REPORT_1_TEST CPU.md @@ -0,0 +1,134 @@ +# Benchmark Report + +## Sample Test Results + +This benchmark report summarizes the performance of tests on the `test/bench/cpu_transforms.ench.js` script, comparing CPU transforms set to true and false. The results show that TEST CPU TRANSFORMS false consistently outperforms true, with higher operations per second (ops/sec) and lower statistical variations. For example, in one test, false achieved 0.9418 ops/sec compared to true at 0.7533 ops/sec, making false 1.25x faster. The report includes detailed metrics and a summary of the relative performance differences. + +https://editor.p5js.org/davepagurek/sketches/FAqbP5k8i + +## Script that we are benchmarking + +Since in the test environment we are not running the draw function we are benchmarking the first run of the draw. + +``` +const TEST_CPU_TRANSFORMS = false + +// Not testing strokes rn because they're slower in general and we don't +// need them for this test to make sense +p5.Geometry.prototype._edgesToVertices = () => {} + +let fps +const state = [] + +function setup() { + createCanvas(400, 400, WEBGL); + for (let i = 0; i < 100; i++) { + state.push({ + pos: createVector(random(-200, 200), random(-200, 200)), + vel: createVector(random(-2, 2), random(-2, 2)), + }) + } + fps = createP() +} + +function draw() { + background(220); + + for (const s of state) { + s.pos.add(s.vel) + for (const axis of ['x', 'y']) { + if (s.pos[axis] < -200) { + s.pos[axis] = -200 + s.vel[axis] *= -1 + } + if (s.pos[axis] > 200) { + s.pos[axis] = 200 + s.vel[axis] *= -1 + } + } + } + + const drawCircles = () => { + for (const s of state) { + push() + translate(s.pos.x, s.pos.y) + const pts = 500 + beginShape(TRIANGLE_FAN) + vertex(0,0) + for (let i = 0; i <= pts; i++) { + const a = (i/pts) * TWO_PI + vertex(5*cos(a), 5*sin(a)) + } + endShape() + pop() + } + } + if (TEST_CPU_TRANSFORMS) { + // Flattens into a single buffer + const shape = buildGeometry(drawCircles) + model(shape) + freeGeometry(model) + } else { + drawCircles() + } + + fps.html(round(frameRate())) +} +``` + + +### test/bench/dave.bench.js (2) 35272ms +- **Dave bench test (2) 35268ms** + - **TEST CPU TRANSFORMS true**: 0.7533 ops/sec ±3.80% (10 samples) + - **TEST CPU TRANSFORMS false**: 0.9418 ops/sec ±0.20% (10 samples) _fastest_ + +#### Summary +- **TEST CPU TRANSFORMS false** is 1.25x faster than **TEST CPU TRANSFORMS true** + +### Separated +#### test/bench/dave.bench.js (2) 27912ms +- **Dave bench test (2) 27909ms** + - **TEST CPU TRANSFORMS false**: 0.5350 ops/sec ±4.39% (10 samples) + - **TEST CPU TRANSFORMS true**: _skipped_ + +#### test/bench/dave.bench.js (2) 19331ms +- **Dave bench test (2) 19327ms** + - **TEST CPU TRANSFORMS false**: _skipped_ + - **TEST CPU TRANSFORMS true**: 0.7519 ops/sec ±3.80% (10 samples) + +### Comparing +#### DEV v2.1.5 /Users/cristianbanuelos/repos/p5.js.git/bench +- **test/bench/dave.bench.js (2) 15671ms** + - **Dave bench test (2) 15667ms** + - **TEST CPU TRANSFORMS false**: 0.9719 ops/sec ±1.36% (2 samples) _fastest_ + - **TEST CPU TRANSFORMS true**: 0.8170 ops/sec ±15.16% (2 samples) + +#### Summary +- **TEST CPU TRANSFORMS false** is 1.19x faster than **TEST CPU TRANSFORMS true** + +#### DEV v2.1.5 /Users/cristianbanuelos/repos/p5.js.git/bench +- **test/bench/dave.bench.js (2) 15915ms** + - **Dave bench test (2) 15905ms** + - **TEST CPU TRANSFORMS true**: 0.7961 ops/sec ±4.86% (2 samples) + - **TEST CPU TRANSFORMS false**: 0.9286 ops/sec ±36.88% (2 samples) _fastest_ + +#### Summary +- **TEST CPU TRANSFORMS false** is 1.17x faster than **TEST CPU TRANSFORMS true** + +## With 1000 Elements +### test/bench/dave.bench.js (2) 151509ms +- **Dave bench test (2) 151505ms** + - **TEST CPU TRANSFORMS true**: 0.0582 ops/sec ±0.00% (1 sample) + - **TEST CPU TRANSFORMS false**: 0.0912 ops/sec ±0.00% (1 sample) _fastest_ + +#### Summary +- **TEST CPU TRANSFORMS false** is 1.57x faster than **TEST CPU TRANSFORMS true** + +### 1000 Elements Second Run +#### test/bench/dave.bench.js (2) 150601ms +- **Dave bench test (2) 150597ms** + - **TEST CPU TRANSFORMS true**: 0.0589 ops/sec ±0.00% (1 sample) + - **TEST CPU TRANSFORMS false**: 0.0914 ops/sec ±0.00% (1 sample) _fastest_ + +#### Summary +- **TEST CPU TRANSFORMS false** is 1.55x faster than **TEST CPU TRANSFORMS true** diff --git a/test/bench/cpu_transforms.bench.js b/test/bench/cpu_transforms.bench.js new file mode 100644 index 0000000000..f94b322842 --- /dev/null +++ b/test/bench/cpu_transforms.bench.js @@ -0,0 +1,168 @@ +// import p5 from "../../../src/app.js"; +import { bench, describe } from "vitest"; +import p5 from "../../src/app"; + +const options = { iterations: 1, time: 1500 }; +const ITERATIONS = 100 +describe.sequential("Dave bench test", () => { + bench( + "TEST CPU TRANSFORMS true", + async () => { + try { + const TEST_CPU_TRANSFORMS = true; + var myp5; + new p5(function (p) { + p.setup = function () { + myp5 = p; + }; + }); + await vi.waitFor(() => { + if (myp5 === undefined) { + throw new Error("not ready"); + } + }); + let fps; + const state = []; + + myp5.createCanvas(400, 400, myp5.WEBGL); + for (let i = 0; i < ITERATIONS; i++) { + state.push({ + pos: myp5.createVector( + myp5.random(-200, 200), + myp5.random(-200, 200) + ), + vel: myp5.createVector(myp5.random(-2, 2), myp5.random(-2, 2)), + }); + } + fps = myp5.createP(); + + assert.equal(myp5.webglVersion, myp5.webglVersion); + myp5.remove(); + + // Now what's in draw + myp5.background(220); + + for (const s of state) { + s.pos.add(s.vel); + for (const axis of ["x", "y"]) { + if (s.pos[axis] < -200) { + s.pos[axis] = -200; + s.vel[axis] *= -1; + } + if (s.pos[axis] > 200) { + s.pos[axis] = 200; + s.vel[axis] *= -1; + } + } + } + + const drawCircles = () => { + for (const s of state) { + myp5.push(); + myp5.translate(s.pos.x, s.pos.y); + const pts = 500; + myp5.beginShape(myp5.TRIANGLE_FAN); + myp5.vertex(0, 0); + for (let i = 0; i <= pts; i++) { + const a = (i / pts) * myp5.TWO_PI; + myp5.vertex(5 * myp5.cos(a), 5 * myp5.sin(a)); + } + myp5.endShape(); + myp5.pop(); + } + }; + + if (TEST_CPU_TRANSFORMS) { + // Flattens into a single buffer + const shape = myp5.buildGeometry(drawCircles); + myp5.model(shape); + myp5.freeGeometry(myp5.model); + } else { + drawCircles(); + } + } catch (error) { + console.log(error); + } + }, + options + ), + options; + bench( + "TEST CPU TRANSFORMS false", + async () => { + const TEST_CPU_TRANSFORMS = false; + var myp5; + new p5(function (p) { + p.setup = function () { + myp5 = p; + }; + }); + await vi.waitFor(() => { + if (myp5 === undefined) { + throw new Error("not ready"); + } + }); + let fps; + const state = []; + + myp5.createCanvas(400, 400, myp5.WEBGL); + for (let i = 0; i < ITERATIONS; i++) { + state.push({ + pos: myp5.createVector( + myp5.random(-200, 200), + myp5.random(-200, 200) + ), + vel: myp5.createVector(myp5.random(-2, 2), myp5.random(-2, 2)), + }); + } + fps = myp5.createP(); + + assert.equal(myp5.webglVersion, myp5.webglVersion); + myp5.remove(); + + // Now what's in draw + myp5.background(220); + + for (const s of state) { + s.pos.add(s.vel); + for (const axis of ["x", "y"]) { + if (s.pos[axis] < -200) { + s.pos[axis] = -200; + s.vel[axis] *= -1; + } + if (s.pos[axis] > 200) { + s.pos[axis] = 200; + s.vel[axis] *= -1; + } + } + } + + const drawCircles = () => { + for (const s of state) { + myp5.push(); + myp5.translate(s.pos.x, s.pos.y); + const pts = 500; + myp5.beginShape(myp5.TRIANGLE_FAN); + myp5.vertex(0, 0); + for (let i = 0; i <= pts; i++) { + const a = (i / pts) * myp5.TWO_PI; + myp5.vertex(5 * myp5.cos(a), 5 * myp5.sin(a)); + } + myp5.endShape(); + myp5.pop(); + } + }; + + if (TEST_CPU_TRANSFORMS) { + // Flattens into a single buffer + const shape = myp5.buildGeometry(drawCircles); + myp5.model(shape); + myp5.freeGeometry(myp5.model); + } else { + drawCircles(); + } + }, + options + ), + options; +}); diff --git a/test/bench/poc.bench_.js b/test/bench/poc.bench_.js new file mode 100644 index 0000000000..40a79d0995 --- /dev/null +++ b/test/bench/poc.bench_.js @@ -0,0 +1,29 @@ +import { bench, describe } from "vitest"; + +describe.skip("sort", () => { + bench( + "normal", + () => { + const x = [1, 5, 4, 2, 3]; + x.sort((a, b) => { + return a - b; + }); + throw new Error("error"); + }, + { iterations: 100 } + ); + + bench( + "reverse", + () => { + const x = [1, 5, 4, 2, 3]; + x.reverse() + .reverse() + .reverse() + .sort((a, b) => { + return a - b; + }); + }, + { iterations: 10 } + ); +}); diff --git a/test/bench/rendering.bench.js b/test/bench/rendering.bench.js new file mode 100644 index 0000000000..4e4c6ece69 --- /dev/null +++ b/test/bench/rendering.bench.js @@ -0,0 +1,199 @@ +// import p5 from "../../../src/app.js"; +import { bench, describe } from "vitest"; +import p5 from "../../src/app"; + +const options = { iterations: 20, time: 500 }; +describe("Rendering bench test", () => { + bench( + "normal", + async () => { + try { + var myp5; + new p5(function (p) { + p.setup = function () { + myp5 = p; + p.rect(10, 10, 10, 10); + }; + }); + await vi.waitFor(() => { + if (myp5 === undefined) { + throw new Error("not ready"); + } + }); + + myp5.createCanvas(13, 15); + myp5.fill(0, 100, 0); + myp5.rect(20, 20, 20, 20); + + assert.equal(myp5.webglVersion, myp5.P2D); + myp5.remove(); + } catch (error) { + console.log(error); + } + }, + options + ); + + bench( + "thousand", + async () => { + try { + var myp5; + new p5(function (p) { + p.setup = function () { + myp5 = p; + p.rect(10, 10, 10, 10); + }; + }); + await vi.waitFor(() => { + if (myp5 === undefined) { + throw new Error("not ready"); + } + }); + + myp5.createCanvas(13, 15); + myp5.fill(0, 100, 0); + myp5.rect(20, 20, 20, 20); + for (let i = 0; i < 1000; i++) { + myp5.rect(20, 20, 20, 20); + } + + assert.equal(myp5.webglVersion, myp5.P2D); + myp5.remove(); + } catch (error) { + console.log(error); + } + }, + options + ); + + bench( + "10k", + async () => { + try { + var myp5; + new p5(function (p) { + p.setup = function () { + myp5 = p; + p.rect(10, 10, 10, 10); + }; + }); + await vi.waitFor(() => { + if (myp5 === undefined) { + throw new Error("not ready"); + } + }); + + myp5.createCanvas(13, 15); + myp5.fill(0, 100, 0); + myp5.rect(20, 20, 20, 20); + for (let i = 0; i < 10000; i++) { + myp5.rect(20, 20, 20, 20); + } + + assert.equal(myp5.webglVersion, myp5.P2D); + myp5.remove(); + } catch (error) { + console.log(error); + } + }, + options + ); +}); +describe("Another suire", () => { + bench( + "normal v2", + async () => { + try { + var myp5; + new p5(function (p) { + p.setup = function () { + myp5 = p; + p.rect(10, 10, 10, 10); + }; + }); + await vi.waitFor(() => { + if (myp5 === undefined) { + throw new Error("not ready"); + } + }); + + myp5.createCanvas(13, 15); + myp5.fill(0, 100, 0); + myp5.rect(20, 20, 20, 20); + + assert.equal(myp5.webglVersion, myp5.P2D); + myp5.remove(); + } catch (error) { + console.log(error); + } + }, + options + ); + + bench( + "thousand v2", + async () => { + try { + var myp5; + new p5(function (p) { + p.setup = function () { + myp5 = p; + p.rect(10, 10, 10, 10); + }; + }); + await vi.waitFor(() => { + if (myp5 === undefined) { + throw new Error("not ready"); + } + }); + + myp5.createCanvas(13, 15); + myp5.fill(0, 100, 0); + myp5.rect(20, 20, 20, 20); + for (let i = 0; i < 1000; i++) { + myp5.rect(20, 20, 20, 20); + } + + assert.equal(myp5.webglVersion, myp5.P2D); + myp5.remove(); + } catch (error) { + console.log(error); + } + }, + options + ); + + bench( + "10k v2", + async () => { + try { + var myp5; + new p5(function (p) { + p.setup = function () { + myp5 = p; + p.rect(10, 10, 10, 10); + }; + }); + await vi.waitFor(() => { + if (myp5 === undefined) { + throw new Error("not ready"); + } + }); + + myp5.createCanvas(13, 15); + myp5.fill(0, 100, 0); + myp5.rect(20, 20, 20, 20); + for (let i = 0; i < 10000; i++) { + myp5.rect(20, 20, 20, 20); + } + + assert.equal(myp5.webglVersion, myp5.P2D); + myp5.remove(); + } catch (error) { + console.log(error); + } + }, + options + ); +}); diff --git a/vitest.workspace.mjs b/vitest.workspace.mjs index c5f497ce64..aa3b7057be 100644 --- a/vitest.workspace.mjs +++ b/vitest.workspace.mjs @@ -17,7 +17,8 @@ export default defineWorkspace([ name: 'unit', root: './', include: [ - './test/unit/**/*.js' + './test/unit/**/*.js', + './test/bench/**/*.js' ], exclude: [ './test/unit/spec.js', From 528e05f01d7807681b8c0cc1f954544373c9e582 Mon Sep 17 00:00:00 2001 From: "Dr. Holomorfo" Date: Wed, 11 Dec 2024 23:24:20 -0800 Subject: [PATCH 2/3] separate bench and test flows configuration. --- vitest.workspace.mjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vitest.workspace.mjs b/vitest.workspace.mjs index aa3b7057be..14bac25ce4 100644 --- a/vitest.workspace.mjs +++ b/vitest.workspace.mjs @@ -13,12 +13,18 @@ export default defineWorkspace([ { plugins, publicDir: './test', + bench: { + name: 'bench', + root: './', + include: [ + './test/bench/**/*.js' + ], + }, test: { name: 'unit', root: './', include: [ './test/unit/**/*.js', - './test/bench/**/*.js' ], exclude: [ './test/unit/spec.js', From 6cb2b194baf4450dda5059a909f663720fcad3ad Mon Sep 17 00:00:00 2001 From: "Dr. Holomorfo" Date: Sun, 15 Dec 2024 22:24:32 -0800 Subject: [PATCH 3/3] update bench placeholder tests --- test/bench/{poc.bench_.js => poc.bench.js} | 2 +- test/bench/rendering.bench.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) rename test/bench/{poc.bench_.js => poc.bench.js} (93%) diff --git a/test/bench/poc.bench_.js b/test/bench/poc.bench.js similarity index 93% rename from test/bench/poc.bench_.js rename to test/bench/poc.bench.js index 40a79d0995..100cd2f197 100644 --- a/test/bench/poc.bench_.js +++ b/test/bench/poc.bench.js @@ -1,6 +1,6 @@ import { bench, describe } from "vitest"; -describe.skip("sort", () => { +describe.only("sort", () => { bench( "normal", () => { diff --git a/test/bench/rendering.bench.js b/test/bench/rendering.bench.js index 4e4c6ece69..7d4813473f 100644 --- a/test/bench/rendering.bench.js +++ b/test/bench/rendering.bench.js @@ -54,7 +54,7 @@ describe("Rendering bench test", () => { myp5.createCanvas(13, 15); myp5.fill(0, 100, 0); myp5.rect(20, 20, 20, 20); - for (let i = 0; i < 1000; i++) { + for (let i = 0; i < 10000; i++) { myp5.rect(20, 20, 20, 20); } @@ -100,7 +100,8 @@ describe("Rendering bench test", () => { options ); }); -describe("Another suire", () => { + +describe("Another suite", () => { bench( "normal v2", async () => {