diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d69fbe60..17725696 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,8 @@ on: branches: - "main" pull_request: + workflow_dispatch: + jobs: check: runs-on: "ubuntu-latest" @@ -20,30 +22,6 @@ jobs: - run: pnpm install --frozen-lockfile --prefer-offline - run: pnpm moon check --all - codspeed: - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@v3" - with: - fetch-depth: 0 - - run: sudo apt-get install -y valgrind - - uses: pnpm/action-setup@v2 - - uses: actions/setup-node@v3 - with: - cache: pnpm - node-version-file: .nvmrc - - run: pnpm install --frozen-lockfile --prefer-offline - - run: pnpm moon run :build - - - name: Run benchmarks - # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v1` - uses: CodSpeedHQ/action@main - with: - run: | - pnpm moon run --concurrency 1 :bench - pnpm --workspace-concurrency 1 -r bench-tinybench - pnpm --workspace-concurrency 1 -r bench-benchmark-js - list-examples: runs-on: "ubuntu-latest" name: List examples diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000..b0b93e56 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,32 @@ +name: "codspeed" +on: + push: + branches: + - "main" + pull_request: + workflow_dispatch: + +jobs: + codspeed: + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@v3" + with: + fetch-depth: 0 + - run: sudo apt-get install -y valgrind + - uses: pnpm/action-setup@v2 + - uses: actions/setup-node@v3 + with: + cache: pnpm + node-version-file: .nvmrc + - run: pnpm install --frozen-lockfile --prefer-offline + - run: pnpm moon run :build + + - name: Run benchmarks + # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v1` + uses: CodSpeedHQ/action@main + with: + run: | + pnpm moon run --concurrency 1 :bench + pnpm --workspace-concurrency 1 -r bench-tinybench + pnpm --workspace-concurrency 1 -r bench-benchmark-js diff --git a/examples/with-javascript-cjs/tinybench.js b/examples/with-javascript-cjs/tinybench.js index ad925f79..46497de0 100644 --- a/examples/with-javascript-cjs/tinybench.js +++ b/examples/with-javascript-cjs/tinybench.js @@ -20,11 +20,5 @@ bench }); bench.run().then(() => { - console.table( - bench.tasks.map(({ name, result }) => ({ - "Task Name": name, - "Average Time (ps)": result?.mean ? result.mean * 1000 : "N/A", - "Variance (ps)": result?.variance ? result.variance * 1000 : "N/A", - })) - ); + console.table(bench.table()); }); diff --git a/examples/with-javascript-esm/tinybench.js b/examples/with-javascript-esm/tinybench.js index a4a302a4..e6736b14 100644 --- a/examples/with-javascript-esm/tinybench.js +++ b/examples/with-javascript-esm/tinybench.js @@ -20,11 +20,5 @@ bench }); bench.run().then(() => { - console.table( - bench.tasks.map(({ name, result }) => ({ - "Task Name": name, - "Average Time (ps)": result?.mean ? result.mean * 1000 : "N/A", - "Variance (ps)": result?.variance ? result.variance * 1000 : "N/A", - })) - ); + console.table(bench.table()); }); diff --git a/examples/with-typescript-cjs/bench/tinybench/index.bench.ts b/examples/with-typescript-cjs/bench/tinybench/index.bench.ts index 556a5982..fbc70534 100644 --- a/examples/with-typescript-cjs/bench/tinybench/index.bench.ts +++ b/examples/with-typescript-cjs/bench/tinybench/index.bench.ts @@ -10,11 +10,5 @@ export const bench = withCodSpeed(new Bench()); registerFoobarbazBenchmarks(bench); await bench.run(); - console.table( - bench.tasks.map(({ name, result }) => ({ - "Task Name": name, - "Average Time (ps)": result?.mean ? result.mean * 1000 : "N/A", - "Variance (ps)": result?.variance ? result.variance * 1000 : "N/A", - })) - ); + console.table(bench.table()); })(); diff --git a/examples/with-typescript-cjs/package.json b/examples/with-typescript-cjs/package.json index b206bc5c..2b9c4566 100644 --- a/examples/with-typescript-cjs/package.json +++ b/examples/with-typescript-cjs/package.json @@ -10,6 +10,7 @@ "@codspeed/tinybench-plugin": "workspace:*", "@types/benchmark": "^2.1.2", "benchmark": "^2.1.4", + "esbuild-register": "^3.4.2", "tinybench": "^2.5.0", "typescript": "^5.1.3" } diff --git a/examples/with-typescript-esm/bench/tinybench/index.bench.ts b/examples/with-typescript-esm/bench/tinybench/index.bench.ts index 556a5982..fbc70534 100644 --- a/examples/with-typescript-esm/bench/tinybench/index.bench.ts +++ b/examples/with-typescript-esm/bench/tinybench/index.bench.ts @@ -10,11 +10,5 @@ export const bench = withCodSpeed(new Bench()); registerFoobarbazBenchmarks(bench); await bench.run(); - console.table( - bench.tasks.map(({ name, result }) => ({ - "Task Name": name, - "Average Time (ps)": result?.mean ? result.mean * 1000 : "N/A", - "Variance (ps)": result?.variance ? result.variance * 1000 : "N/A", - })) - ); + console.table(bench.table()); })(); diff --git a/examples/with-typescript-esm/package.json b/examples/with-typescript-esm/package.json index 6548d03a..d0e15a8f 100644 --- a/examples/with-typescript-esm/package.json +++ b/examples/with-typescript-esm/package.json @@ -11,6 +11,7 @@ "@codspeed/tinybench-plugin": "workspace:*", "@types/benchmark": "^2.1.2", "benchmark": "^2.1.4", + "esbuild-register": "^3.4.2", "tinybench": "^2.5.0", "typescript": "^5.1.3" } diff --git a/examples/with-typescript-simple-cjs/tinybench.ts b/examples/with-typescript-simple-cjs/tinybench.ts index a4a302a4..e6736b14 100644 --- a/examples/with-typescript-simple-cjs/tinybench.ts +++ b/examples/with-typescript-simple-cjs/tinybench.ts @@ -20,11 +20,5 @@ bench }); bench.run().then(() => { - console.table( - bench.tasks.map(({ name, result }) => ({ - "Task Name": name, - "Average Time (ps)": result?.mean ? result.mean * 1000 : "N/A", - "Variance (ps)": result?.variance ? result.variance * 1000 : "N/A", - })) - ); + console.table(bench.table()); }); diff --git a/examples/with-typescript-simple-esm/tinybench.ts b/examples/with-typescript-simple-esm/tinybench.ts index a4a302a4..e6736b14 100644 --- a/examples/with-typescript-simple-esm/tinybench.ts +++ b/examples/with-typescript-simple-esm/tinybench.ts @@ -20,11 +20,5 @@ bench }); bench.run().then(() => { - console.table( - bench.tasks.map(({ name, result }) => ({ - "Task Name": name, - "Average Time (ps)": result?.mean ? result.mean * 1000 : "N/A", - "Variance (ps)": result?.variance ? result.variance * 1000 : "N/A", - })) - ); + console.table(bench.table()); }); diff --git a/lerna.json b/lerna.json index 0a500211..7816eacf 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "useWorkspaces": true, "packages": ["packages/*"], "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.1.0" + "version": "2.2.0" } diff --git a/packages/benchmark.js-plugin/package.json b/packages/benchmark.js-plugin/package.json index 52c1857c..dd9e377d 100644 --- a/packages/benchmark.js-plugin/package.json +++ b/packages/benchmark.js-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@codspeed/benchmark.js-plugin", - "version": "2.1.0", + "version": "2.2.0", "description": "Benchmark.js compatibility layer for CodSpeed", "keywords": [ "codspeed", @@ -28,7 +28,7 @@ "jest-mock-extended": "^3.0.4" }, "dependencies": { - "@codspeed/core": "workspace:^2.1.0", + "@codspeed/core": "workspace:^2.2.0", "find-up": "^6.3.0", "lodash": "^4.17.10", "stack-trace": "1.0.0-pre2" diff --git a/packages/benchmark.js-plugin/src/index.ts b/packages/benchmark.js-plugin/src/index.ts index a2d06c13..a7442a56 100644 --- a/packages/benchmark.js-plugin/src/index.ts +++ b/packages/benchmark.js-plugin/src/index.ts @@ -183,6 +183,10 @@ async function runBenchmarks({ benchPayload = bench.fn as CallableFunction; } + if (typeof bench.options.setup === "function") { + await bench.options.setup(); + } + if (isAsync) { await optimizeFunction(benchPayload); await (async function __codspeed_root_frame__() { @@ -198,9 +202,14 @@ async function runBenchmarks({ Measurement.stopInstrumentation(uri); })(); } + + if (typeof bench.options.teardown === "function") { + await bench.options.teardown(); + } + console.log(` ✔ Measured ${uri}`); benchmarkCompletedListeners.forEach((listener) => listener()); - teardownCore(); } + teardownCore(); console.log(`[CodSpeed] Done running ${benches.length} benches.`); } diff --git a/packages/benchmark.js-plugin/tests/index.integ.test.ts b/packages/benchmark.js-plugin/tests/index.integ.test.ts index b70d8c86..27087984 100644 --- a/packages/benchmark.js-plugin/tests/index.integ.test.ts +++ b/packages/benchmark.js-plugin/tests/index.integ.test.ts @@ -1,16 +1,13 @@ import { mockDeep, mockReset } from "jest-mock-extended"; -const mockCore = mockDeep(); +const mockCore = mockDeep(); -import type { Measurement } from "@codspeed/core"; +import * as core from "@codspeed/core"; import Benchmark from "benchmark"; import { withCodSpeed } from ".."; import { registerBenchmarks } from "./registerBenchmarks"; import { registerOtherBenchmarks } from "./registerOtherBenchmarks"; -jest.mock("@codspeed/core", () => ({ - ...jest.requireActual("@codspeed/core"), - Measurement: mockCore, -})); +jest.mock("@codspeed/core", () => mockCore); beforeEach(() => { mockReset(mockCore); @@ -23,7 +20,7 @@ const benchOptions: Benchmark.Options = { describe("Benchmark", () => { it("simple benchmark", async () => { - mockCore.isInstrumented.mockReturnValue(false); + mockCore.Measurement.isInstrumented.mockReturnValue(false); const bench = withCodSpeed( new Benchmark( "RegExp", @@ -37,11 +34,11 @@ describe("Benchmark", () => { bench.on("complete", onComplete); await bench.run(); expect(onComplete).toHaveBeenCalled(); - expect(mockCore.startInstrumentation).not.toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).not.toHaveBeenCalled(); + expect(mockCore.Measurement.startInstrumentation).not.toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).not.toHaveBeenCalled(); }); it("check core methods are called", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const bench = withCodSpeed( new Benchmark( @@ -56,13 +53,13 @@ describe("Benchmark", () => { bench.on("complete", onComplete); await bench.run(); expect(onComplete).toHaveBeenCalled(); - expect(mockCore.startInstrumentation).toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/benchmark.js-plugin/tests/index.integ.test.ts::RegExpSingle" ); }); it("check error handling", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const bench = withCodSpeed( new Benchmark( "throwing", @@ -79,7 +76,7 @@ describe("Benchmark", () => { async (instrumented) => { const logSpy = jest.spyOn(console, "log"); const warnSpy = jest.spyOn(console, "warn"); - mockCore.isInstrumented.mockReturnValue(instrumented); + mockCore.Measurement.isInstrumented.mockReturnValue(instrumented); await withCodSpeed( new Benchmark( "RegExpSingle", @@ -107,11 +104,28 @@ describe("Benchmark", () => { } } ); + it("should call setup and teardown", async () => { + mockCore.Measurement.isInstrumented.mockReturnValue(true); + const setup = jest.fn(); + const teardown = jest.fn(); + const bench = withCodSpeed( + new Benchmark( + "RegExpSingle", + function () { + /o/.test("Hello World!"); + }, + { ...benchOptions, setup, teardown } + ) + ); + await bench.run(); + expect(setup).toHaveBeenCalled(); + expect(teardown).toHaveBeenCalled(); + }); }); describe("Benchmark.Suite", () => { it("simple suite", async () => { - mockCore.isInstrumented.mockReturnValue(false); + mockCore.Measurement.isInstrumented.mockReturnValue(false); const suite = withCodSpeed(new Benchmark.Suite()); suite.add( "RegExp", @@ -124,11 +138,11 @@ describe("Benchmark.Suite", () => { suite.on("complete", onComplete); await suite.run({ maxTime: 0.1, initCount: 1 }); expect(onComplete).toHaveBeenCalled(); - expect(mockCore.startInstrumentation).not.toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).not.toHaveBeenCalled(); + expect(mockCore.Measurement.startInstrumentation).not.toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).not.toHaveBeenCalled(); }); it("check core methods are called", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const suite = withCodSpeed(new Benchmark.Suite()).add( "RegExp", function () { @@ -139,13 +153,13 @@ describe("Benchmark.Suite", () => { const onComplete = jest.fn(); suite.on("complete", onComplete); await suite.run({ maxTime: 0.1, initCount: 1 }); - expect(mockCore.startInstrumentation).toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/benchmark.js-plugin/tests/index.integ.test.ts::RegExp" ); }); it("check suite name is in the uri", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); await withCodSpeed(new Benchmark.Suite("thesuite")) .add( "RegExp", @@ -158,15 +172,15 @@ describe("Benchmark.Suite", () => { /o/.test("Hello World!"); }, benchOptions) .run(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/benchmark.js-plugin/tests/index.integ.test.ts::thesuite::RegExp" ); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/benchmark.js-plugin/tests/index.integ.test.ts::thesuite::unknown_1" ); }); it("check error handling", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const bench = withCodSpeed(new Benchmark.Suite("thesuite")).add( "throwing", () => { @@ -180,7 +194,7 @@ describe("Benchmark.Suite", () => { async (instrumented) => { const logSpy = jest.spyOn(console, "log"); const warnSpy = jest.spyOn(console, "warn"); - mockCore.isInstrumented.mockReturnValue(instrumented); + mockCore.Measurement.isInstrumented.mockReturnValue(instrumented); await withCodSpeed(new Benchmark.Suite("thesuite")) .add( "RegExp", @@ -212,31 +226,62 @@ describe("Benchmark.Suite", () => { } ); it("check nested file path is in the uri when bench is registered in another file", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const suite = withCodSpeed(new Benchmark.Suite("thesuite")); registerBenchmarks(suite); const onComplete = jest.fn(); suite.on("complete", onComplete); await suite.run({ maxTime: 0.1, initCount: 1 }); - expect(mockCore.startInstrumentation).toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/benchmark.js-plugin/tests/registerBenchmarks.ts::thesuite::RegExp" ); }); it("check that benchmarks with same name have different URIs when registered in different files", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const suite = withCodSpeed(new Benchmark.Suite("thesuite")); registerBenchmarks(suite); registerOtherBenchmarks(suite); const onComplete = jest.fn(); suite.on("complete", onComplete); await suite.run({ maxTime: 0.1, initCount: 1 }); - expect(mockCore.startInstrumentation).toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/benchmark.js-plugin/tests/registerBenchmarks.ts::thesuite::RegExp" ); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/benchmark.js-plugin/tests/registerOtherBenchmarks.ts::thesuite::RegExp" ); }); + it("should call setupCore and teardownCore only once after run()", async () => { + mockCore.Measurement.isInstrumented.mockReturnValue(true); + const suite = withCodSpeed(new Benchmark.Suite("thesuite")); + registerBenchmarks(suite); + registerOtherBenchmarks(suite); + + expect(mockCore.setupCore).not.toHaveBeenCalled(); + expect(mockCore.teardownCore).not.toHaveBeenCalled(); + + await suite.run({ maxTime: 0.1, initCount: 1 }); + + expect(mockCore.setupCore).toHaveBeenCalledTimes(1); + expect(mockCore.teardownCore).toHaveBeenCalledTimes(1); + }); + it("should call setup and teardown", async () => { + mockCore.Measurement.isInstrumented.mockReturnValue(true); + const setup = jest.fn(); + const teardown = jest.fn(); + + const suite = withCodSpeed(new Benchmark.Suite("thesuite")).add( + "RegExpSingle", + function () { + /o/.test("Hello World!"); + }, + { ...benchOptions, setup, teardown } + ); + await suite.run(); + + expect(setup).toHaveBeenCalled(); + expect(teardown).toHaveBeenCalled(); + }); }); diff --git a/packages/core/package.json b/packages/core/package.json index dcfb6767..332df3ee 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@codspeed/core", - "version": "2.1.0", + "version": "2.2.0", "description": "The core Node library used to integrate with Codspeed runners", "keywords": [ "codspeed", diff --git a/packages/tinybench-plugin/README.md b/packages/tinybench-plugin/README.md index 0d361261..0e17300f 100644 --- a/packages/tinybench-plugin/README.md +++ b/packages/tinybench-plugin/README.md @@ -59,12 +59,7 @@ bench }); await bench.run(); -console.table( - bench.tasks.map(({ name, result }) => ({ - "Task Name": name, - "Average Time (ps)": result?.mean * 1000, - })) -); +console.table(bench.table()); ``` Here, a few things are happening: @@ -81,12 +76,12 @@ $ node benches/bench.mjs [CodSpeed] 2 benches detected but no instrumentation found [CodSpeed] falling back to tinybench -┌─────────┬───────────────┬────────────────────┐ -│ (index) │ Task Name │ Average Time (ps) │ -├─────────┼───────────────┼────────────────────┤ -│ 0 │ 'fibonacci10' │ 0.5660083779532603 │ -│ 1 │ 'fibonacci15' │ 5.2475729101797635 │ -└─────────┴───────────────┴────────────────────┘ +┌─────────┬───────────────┬─────────────┬───────────────────┬──────────┬─────────┐ +│ (index) │ Task Name │ ops/sec │ Average Time (ns) │ Margin │ Samples │ +├─────────┼───────────────┼─────────────┼───────────────────┼──────────┼─────────┤ +│ 0 │ 'fibonacci10' │ '1,810,236' │ 552.4139857896414 │ '±0.18%' │ 905119 │ +│ 1 │ 'fibonacci15' │ '177,516' │ 5633.276191749634 │ '±0.14%' │ 88759 │ +└─────────┴───────────────┴─────────────┴───────────────────┴──────────┴─────────┘ ``` And... Congrats🎉, CodSpeed is installed in your benchmarking suite! Locally, CodSpeed will fallback to tinybench since the instrumentation is only available in the CI environment for now. diff --git a/packages/tinybench-plugin/benches/sample.ts b/packages/tinybench-plugin/benches/sample.ts index 275ca8c0..664cadea 100644 --- a/packages/tinybench-plugin/benches/sample.ts +++ b/packages/tinybench-plugin/benches/sample.ts @@ -20,11 +20,5 @@ bench }); bench.run().then(() => { - console.table( - bench.tasks.map(({ name, result }) => ({ - "Task Name": name, - "Average Time (ps)": result?.mean ? result.mean * 1000 : "N/A", - "Variance (ps)": result?.variance ? result.variance * 1000 : "N/A", - })) - ); + console.table(bench.table()); }); diff --git a/packages/tinybench-plugin/package.json b/packages/tinybench-plugin/package.json index c6995004..e64657b9 100644 --- a/packages/tinybench-plugin/package.json +++ b/packages/tinybench-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@codspeed/tinybench-plugin", - "version": "2.1.0", + "version": "2.2.0", "description": "tinybench compatibility layer for CodSpeed", "keywords": [ "codspeed", @@ -22,10 +22,10 @@ "@types/find-up": "^4.0.0", "@types/stack-trace": "^0.0.30", "jest-mock-extended": "^3.0.4", - "tinybench": "^2.4.0" + "tinybench": "^2.5.0" }, "dependencies": { - "@codspeed/core": "workspace:^2.1.0", + "@codspeed/core": "workspace:^2.2.0", "find-up": "^6.3.0", "stack-trace": "1.0.0-pre2" }, diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 1d19508c..645889fb 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -44,43 +44,40 @@ export function withCodSpeed(bench: Bench): Bench { const rootCallingFile = getCallingFile(); bench.run = async () => { - setupCore(); console.log(`[CodSpeed] running with @codspeed/tinybench v${__VERSION__}`); + setupCore(); for (const task of bench.tasks) { - // run before hooks - if (task.opts.beforeAll != null) { - await task.opts.beforeAll.call(task); - } - if (task.opts.beforeEach != null) { - await task.opts.beforeEach.call(task); - } - - // run the actual benchmark, with instrumentation const uri = isCodSpeedBenchOptions(task.opts) ? task.opts.uri : `${rootCallingFile}::${task.name}`; - await optimizeFunction(task.fn); + + await task.opts.beforeAll?.call(task); + + // run optimizations + await optimizeFunction(async () => { + await task.opts.beforeEach?.call(task); + await task.fn(); + await task.opts.afterEach?.call(task); + }); + + // run instrumented benchmark + await task.opts.beforeEach?.call(task); await (async function __codspeed_root_frame__() { Measurement.startInstrumentation(); await task.fn(); Measurement.stopInstrumentation(uri); })(); + await task.opts.afterEach?.call(task); - // run after hooks - if (task.opts.afterEach != null) { - await task.opts.afterEach.call(task); - } - if (task.opts.afterAll != null) { - await task.opts.afterAll.call(task); - } + await task.opts.afterAll?.call(task); // print results console.log(` ✔ Measured ${uri}`); } + teardownCore(); console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); return bench.tasks; }; - teardownCore(); return bench; } diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index ffae47ae..192b446e 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -1,16 +1,13 @@ import { mockDeep, mockReset } from "jest-mock-extended"; -const mockCore = mockDeep(); +const mockCore = mockDeep(); -import type { Measurement } from "@codspeed/core"; +import * as core from "@codspeed/core"; import { Bench } from "tinybench"; import { withCodSpeed } from ".."; import { registerBenchmarks } from "./registerBenchmarks"; import { registerOtherBenchmarks } from "./registerOtherBenchmarks"; -jest.mock("@codspeed/core", () => ({ - ...jest.requireActual("@codspeed/core"), - Measurement: mockCore, -})); +jest.mock("@codspeed/core", () => mockCore); beforeEach(() => { mockReset(mockCore); @@ -19,7 +16,7 @@ beforeEach(() => { describe("Benchmark.Suite", () => { it("simple suite", async () => { - mockCore.isInstrumented.mockReturnValue(false); + mockCore.Measurement.isInstrumented.mockReturnValue(false); const bench = withCodSpeed(new Bench({ time: 100 })); const onComplete = jest.fn(); bench.add("RegExp", function () { @@ -29,23 +26,23 @@ describe("Benchmark.Suite", () => { await bench.run(); expect(onComplete).toHaveBeenCalled(); - expect(mockCore.startInstrumentation).not.toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).not.toHaveBeenCalled(); + expect(mockCore.Measurement.startInstrumentation).not.toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).not.toHaveBeenCalled(); }); it("check core methods are called", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); await withCodSpeed(new Bench()) .add("RegExp", function () { /o/.test("Hello World!"); }) .run(); - expect(mockCore.startInstrumentation).toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp" ); }); it("check suite name is in the uri", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); await withCodSpeed(new Bench()) .add("RegExp", function () { /o/.test("Hello World!"); @@ -54,15 +51,15 @@ describe("Benchmark.Suite", () => { /o/.test("Hello World!"); }) .run(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp" ); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2" ); }); it("check error handling", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const bench = withCodSpeed(new Bench()); bench.add("throwing", async () => { throw new Error("test"); @@ -74,7 +71,7 @@ describe("Benchmark.Suite", () => { async (instrumented) => { const logSpy = jest.spyOn(console, "log"); const warnSpy = jest.spyOn(console, "warn"); - mockCore.isInstrumented.mockReturnValue(instrumented); + mockCore.Measurement.isInstrumented.mockReturnValue(instrumented); await withCodSpeed(new Bench({ time: 100 })) .add("RegExp", function () { /o/.test("Hello World!"); @@ -103,12 +100,12 @@ describe("Benchmark.Suite", () => { } ); it("check nested file path is in the uri when bench is registered in another file", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const bench = withCodSpeed(new Bench()); registerBenchmarks(bench); await bench.run(); - expect(mockCore.startInstrumentation).toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/tinybench-plugin/tests/registerBenchmarks.ts::RegExp" ); }); @@ -117,23 +114,26 @@ describe("Benchmark.Suite", () => { it.failing( "check that benchmarks with same name have different URIs when registered in different files", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); const bench = withCodSpeed(new Bench()); registerBenchmarks(bench); registerOtherBenchmarks(bench); await bench.run(); - expect(mockCore.startInstrumentation).toHaveBeenCalled(); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.startInstrumentation).toHaveBeenCalled(); + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/tinybench-plugin/tests/registerBenchmarks.ts::RegExp" ); - expect(mockCore.stopInstrumentation).toHaveBeenCalledWith( + expect(mockCore.Measurement.stopInstrumentation).toHaveBeenCalledWith( "packages/tinybench-plugin/tests/registerOtherBenchmarks.ts::RegExp" ); } ); it("should run before and after hooks", async () => { - mockCore.isInstrumented.mockReturnValue(true); + mockCore.Measurement.isInstrumented.mockReturnValue(true); + mockCore.optimizeFunction.mockImplementation(async (fn) => { + await fn(); + }); const beforeAll = jest.fn(); const beforeEach = jest.fn(); const afterEach = jest.fn(); @@ -156,9 +156,30 @@ describe("Benchmark.Suite", () => { ) .run(); + // since the optimization is running the benchmark once before the actual run, the each hooks are called twice + expect(beforeEach).toHaveBeenCalledTimes(4); + expect(afterEach).toHaveBeenCalledTimes(4); + expect(beforeAll).toHaveBeenCalledTimes(2); - expect(beforeEach).toHaveBeenCalledTimes(2); - expect(afterEach).toHaveBeenCalledTimes(2); expect(afterAll).toHaveBeenCalledTimes(2); }); + + it("should call setupCore and teardownCore only once after run()", async () => { + mockCore.Measurement.isInstrumented.mockReturnValue(true); + const bench = withCodSpeed(new Bench()) + .add("RegExp", function () { + /o/.test("Hello World!"); + }) + .add("RegExp2", () => { + /o/.test("Hello World!"); + }); + + expect(mockCore.setupCore).not.toHaveBeenCalled(); + expect(mockCore.teardownCore).not.toHaveBeenCalled(); + + await bench.run(); + + expect(mockCore.setupCore).toHaveBeenCalledTimes(1); + expect(mockCore.teardownCore).toHaveBeenCalledTimes(1); + }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eb59792b..40ad6e28 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -136,6 +136,9 @@ importers: benchmark: specifier: ^2.1.4 version: 2.1.4 + esbuild-register: + specifier: ^3.4.2 + version: 3.4.2(esbuild@0.17.16) tinybench: specifier: ^2.5.0 version: 2.5.0 @@ -157,6 +160,9 @@ importers: benchmark: specifier: ^2.1.4 version: 2.1.4 + esbuild-register: + specifier: ^3.4.2 + version: 3.4.2(esbuild@0.17.16) tinybench: specifier: ^2.5.0 version: 2.5.0 @@ -215,7 +221,7 @@ importers: packages/benchmark.js-plugin: dependencies: "@codspeed/core": - specifier: workspace:^2.1.0 + specifier: workspace:^2.2.0 version: link:../core find-up: specifier: ^6.3.0 @@ -268,7 +274,7 @@ importers: packages/tinybench-plugin: dependencies: "@codspeed/core": - specifier: workspace:^2.1.0 + specifier: workspace:^2.2.0 version: link:../core find-up: specifier: ^6.3.0 @@ -287,8 +293,8 @@ importers: specifier: ^3.0.4 version: 3.0.4(jest@29.5.0)(typescript@5.1.3) tinybench: - specifier: ^2.4.0 - version: 2.4.0 + specifier: ^2.5.0 + version: 2.5.0 packages: /@ampproject/remapping@2.2.1: @@ -12215,13 +12221,6 @@ packages: globrex: 0.1.2 dev: true - /tinybench@2.4.0: - resolution: - { - integrity: sha512-iyziEiyFxX4kyxSp+MtY1oCH/lvjH3PxFN8PGCDeqcZWAJ/i+9y+nL85w99PxVzrIvew/GSkSbDYtiGVa85Afg==, - } - dev: true - /tinybench@2.5.0: resolution: { pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy