feat: only run 1 time when not in scheduled job (#28)

* feat: only run 1 time when not in scheduled job

* feat: can config compareMetrics

* feat: can config compareMetrics

Author: SyMind

.github/workflows/scheduled_bench.yml

@@ -11,6 +11,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Init env
         uses: ./.github/actions/env
+      - name: Set Scheduled env
+        shell: bash
+        run: echo "SCHEDULED_JOB=true" >> $GITHUB_ENV
       - name: Build rspack
         run: node bin/cli.js build
       - name: Run benchmark

bench.config.js

@@ -1,17 +1,71 @@
+const isScheduled = !!process.env.SCHEDULED_JOB;
+
+// HMR will run 10 times in build plugin, so we should not start multiple instances of Rspack.
+// However, we still need to run multiple instances of Rspack when executing scheduled tasks for longer runtimes.
+const hmrRuns = isScheduled ? 10 : 1;
+
 export default {
 	jobs: [
-		"10000_development-mode",
-		"10000_development-mode_hmr",
-		"10000_production-mode",
-		"arco-pro_development-mode",
-		"arco-pro_development-mode_intercept-plugin",
-		"arco-pro_development-mode_hmr",
-		"arco-pro_development-mode_hmr_intercept-plugin",
-		"arco-pro_production-mode",
-		"arco-pro_production-mode_intercept-plugin",
-		"threejs_development-mode_10x",
-		"threejs_development-mode_10x_hmr",
-		"threejs_production-mode_10x"
+		{
+			name: "10000_development-mode",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "10000_development-mode_hmr",
+			runs: hmrRuns,
+			compareMetrics: ["stats"]
+		},
+		{
+			name: "10000_production-mode",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "arco-pro_development-mode",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "arco-pro_development-mode_intercept-plugin",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "arco-pro_development-mode_hmr",
+			runs: hmrRuns,
+			compareMetrics: ["stats"]
+		},
+		{
+			name: "arco-pro_development-mode_hmr_intercept-plugin",
+			runs: hmrRuns,
+			compareMetrics: ["stats"]
+		},
+		{
+			name: "arco-pro_production-mode",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "arco-pro_production-mode_intercept-plugin",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "threejs_development-mode_10x",
+			runs: 10,
+			compareMetrics: ["exec"]
+		},
+		{
+			name: "threejs_development-mode_10x_hmr",
+			runs: hmrRuns,
+			compareMetrics: ["stats"]
+		},
+		{
+			name: "threejs_production-mode_10x",
+			runs: 10,
+			compareMetrics: ["exec"]
+		}
 	],
 	rspackDirectory: process.env.RSPACK_DIR
 };

bin/cli.js

@@ -68,12 +68,13 @@ const {
 
 const cwd = process.cwd();
 
-const configPath = join(process.cwd(), "bench.config.js");
-const config = (await import(configPath)).default;
+const benchConfigPath = join(process.cwd(), "bench.config.js");
+const benchConfig = (await import(benchConfigPath)).default;
 
-const jobs = config.jobs ?? [];
-const rspackDirectory = config.rspackDirectory ?? join(cwd, ".rspack");
-const benchmarkDirectory = config.benchmarkDirectory ?? join(cwd, "output");
+const jobs = benchConfig.jobs ?? [];
+const rspackDirectory = benchConfig.rspackDirectory ?? join(cwd, ".rspack");
+const benchmarkDirectory =
+	benchConfig.benchmarkDirectory ?? join(cwd, "output");
 
 if (!command || command === "build") {
 	const fetchUrl = `https://github.com/${repository}`;
@@ -125,29 +126,30 @@ if (!command || command === "bench") {
 		console.log(
 			[
 				`Running jobs for shard ${currentIndex}/${totalShards}:`,
-				...shardJobs
+				...shardJobs.map(job => job.name)
 			].join("\n  * ")
 		);
 
 		for (const job of shardJobs) {
 			const start = Date.now();
-			const result = await run(job);
+			const result = await run(job.name, job.runs);
+			const message = `${job.name} was run ${job.runs} times, with the following results:`;
 			if (isGitHubActions) {
-				actionsCore.startGroup(`${job} result is`);
+				actionsCore.startGroup(message);
 			} else {
-				console.log(`${job} result is`);
+				console.log(message);
 			}
 
 			console.log(formatResultTable(result, { verbose: true }));
 
 			if (isGitHubActions) {
 				actionsCore.endGroup();
 				const cost = Math.ceil((Date.now() - start) / 1000);
-				console.log(`Cost for \`${job}\`: ${cost} s`);
+				console.log(`Cost for \`${job.name}\`: ${cost} s`);
 			}
 
 			await writeFile(
-				join(benchmarkDirectory, `${job}.json`),
+				join(benchmarkDirectory, `${job.name}.json`),
 				JSON.stringify(result, null, 2)
 			);
 		}
@@ -157,5 +159,5 @@ if (!command || command === "bench") {
 }
 
 if (!command || command === "compare") {
-	compare(base, current, benchmarkDirectory);
+	compare(base, current, benchmarkDirectory, jobs);
 }

docs/index.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html>
 	<head>
 		<title>rspack/benchmark</title>

docs/index.js

@@ -329,8 +329,8 @@ class BenchmarkChart {
 									context.dataset.yAxisID === "size"
 										? formatSize(value, value)
 										: context.dataset.yAxisID === "ratio"
-										? formatRatio(value, value)
-										: formatTime(value, value);
+											? formatRatio(value, value)
+											: formatTime(value, value);
 								return `${context.dataset.label}: ${text}`;
 							}
 						}

lib/addons/hmr.js

@@ -3,7 +3,6 @@ import { Addon } from "./common.js";
 export default class extends Addon {
 	async afterSetup(ctx) {
 		ctx.rspackArgs.push("--watch");
-		ctx.runTimes = 5;
 		ctx.config =
 			ctx.config +
 			`

lib/bench.js

@@ -5,7 +5,7 @@ import { useAddons, dirExist } from "./utils.js";
 
 const dirname = path.resolve(fileURLToPath(import.meta.url), "..");
 
-export async function run(benchmarkName) {
+export async function run(benchmarkName, runs) {
 	const [caseName, ...addonNames] = benchmarkName.split("_");
 	const scenario = getScenario(caseName);
 	const addons = await Promise.all(
@@ -20,6 +20,8 @@ export async function run(benchmarkName) {
 
 	await useAddons(addons, "beforeSetup");
 	const ctx = await scenario.setup();
+	ctx.runTimes = runs;
+
 	await useAddons(addons, "afterSetup", ctx);
 
 	try {

lib/compare.js

@@ -71,9 +71,7 @@ async function getResults(date, index, benchmarkDirectory) {
 	);
 }
 
-export async function compare(base, current, benchmarkDirectory) {
-	const compareMetric = ["exec"];
-
+export async function compare(base, current, benchmarkDirectory, jobs) {
 	const index = await fetchIndex();
 	if (base === "latest") {
 		base = index[index.length - 1].date;
@@ -86,13 +84,20 @@ export async function compare(base, current, benchmarkDirectory) {
 	]);
 	const baseData = {};
 	const currentData = {};
-	for (const metric of compareMetric) {
-		for (const { name, result } of baseResults) {
+
+	for (const { name, result } of baseResults) {
+		const job = jobs.find(job => job.name === name);
+		const compareMetrics = job ? job.compareMetrics : [];
+		for (const metric of compareMetrics) {
 			const tag = `${name} + ${metric}`;
 			baseData[tag] = result[metric];
 		}
+	}
 
-		for (const { name, result } of currentResults) {
+	for (const { name, result } of currentResults) {
+		const job = jobs.find(job => job.name === name);
+		const compareMetrics = job ? job.compareMetrics : [];
+		for (const metric of compareMetrics) {
 			const tag = `${name} + ${metric}`;
 			currentData[tag] = result[metric];
 		}

lib/display.js

@@ -135,7 +135,7 @@ export function formatResultTable(result, { verbose, limit, threshold }) {
 					con: l => f(l.name, l.confidence),
 					con: l => f(l.name, l.confidence),
 					n: l => `${l.count}`
-			  }
+				}
 			: undefined)
 	};
 	return formatTable(entries, columns);

lib/utils.js

@@ -21,7 +21,7 @@ export async function runCommand(
 			? {
 					...process.env,
 					...env
-			  }
+				}
 			: undefined
 	});
 	if (hasOnData) {