diff --git a/benchmark/_cli.js b/benchmark/_cli.js index 583da8f6b20622..2becd7368dc610 100644 --- a/benchmark/_cli.js +++ b/benchmark/_cli.js @@ -24,6 +24,7 @@ function CLI(usage, settings) { this.optional = {}; this.items = []; this.test = false; + this.coverage = false; for (const argName of settings.arrayArgs) { this.optional[argName] = []; @@ -66,6 +67,14 @@ function CLI(usage, settings) { mode = 'both'; } else if (arg === 'test') { this.test = true; + } else if (arg === 'coverage') { + this.coverage = true; + // TODO: add support to those benchmarks + const excludedBenchmarks = ['napi', 'http']; + this.items = Object.keys(benchmarks) + .filter((b) => !excludedBenchmarks.includes(b)); + // Run once + this.optional.set = ['n=1']; } else if (['both', 'item'].includes(mode)) { // item arguments this.items.push(arg); @@ -140,8 +149,8 @@ CLI.prototype.getCpuCoreSetting = function() { const isValid = /^(\d+(-\d+)?)(,\d+(-\d+)?)*$/.test(value); if (!isValid) { throw new Error(` - Invalid CPUSET format: "${value}". Please use a single core number (e.g., "0"), - a range of cores (e.g., "0-3"), or a list of cores/ranges + Invalid CPUSET format: "${value}". Please use a single core number (e.g., "0"), + a range of cores (e.g., "0-3"), or a list of cores/ranges (e.g., "0,2,4" or "0-2,4").\n\n${this.usage} `); } diff --git a/benchmark/coverage.js b/benchmark/coverage.js new file mode 100644 index 00000000000000..582ccd0a0f2ac3 --- /dev/null +++ b/benchmark/coverage.js @@ -0,0 +1,117 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const Mod = require('node:module'); + +const benchmarkFolder = __dirname; +const dir = fs.readdirSync(path.join(__dirname, '../lib')); + +const allModuleExports = {}; + +function getCallSite() { + const originalStackFormatter = Error.prepareStackTrace; + Error.prepareStackTrace = (err, stack) => { + // Some benchmarks change the stackTraceLimit if so, get the last line + // TODO: check if it matches the benchmark folder + if (stack.length >= 2) { + return `${stack[2].getFileName()}:${stack[2].getLineNumber()}`; + } + return stack; + }; + + const err = new Error(); + err.stack; // eslint-disable-line no-unused-expressions + Error.prepareStackTrace = originalStackFormatter; + return err.stack; +} + +const skippedFunctionClasses = [ + 'EventEmitter', + 'Worker', + 'ClientRequest', + 'Readable', + 'StringDecoder', + 'TLSSocket', + 'MessageChannel', +]; + +const skippedModules = [ + 'node:cluster', + 'node:trace_events', + 'node:stream/promises', +]; + +function fetchModules(allModuleExports) { + for (const f of dir) { + if (f.endsWith('.js') && !f.startsWith('_')) { + const moduleName = `node:${f.slice(0, f.length - 3)}`; + if (skippedModules.includes(moduleName)) { + continue; + } + const exports = require(moduleName); + allModuleExports[moduleName] = Object.assign({}, exports); + + for (const fnKey of Object.keys(exports)) { + if (typeof exports[fnKey] === 'function' && !fnKey.startsWith('_')) { + if ( + exports[fnKey].toString().match(/^class/) || + skippedFunctionClasses.includes(fnKey) + ) { + // Skip classes for now + continue; + } + const originalFn = exports[fnKey]; + allModuleExports[moduleName][fnKey] = function() { + const callerStr = getCallSite(); + if (typeof callerStr === 'string' && callerStr.startsWith(benchmarkFolder) && + callerStr.replace(benchmarkFolder, '').match(/^\/.+\/.+/)) { + if (!allModuleExports[moduleName][fnKey]._called) { + allModuleExports[moduleName][fnKey]._called = 0; + } + allModuleExports[moduleName][fnKey]._called++; + + + if (!allModuleExports[moduleName][fnKey]._calls) { + allModuleExports[moduleName][fnKey]._calls = []; + } + allModuleExports[moduleName][fnKey]._calls.push(callerStr); + } + return originalFn.apply(exports, arguments); + }; + } + } + } + } +} + +fetchModules(allModuleExports); + +const req = Mod.prototype.require; +Mod.prototype.require = function(id) { + let newId = id; + if (!id.startsWith('node:')) { + newId = `node:${id}`; + } + const data = allModuleExports[newId]; + if (!data) { + return req.apply(this, arguments); + } + return data; +}; + +process.on('beforeExit', () => { + for (const module of Object.keys(allModuleExports)) { + for (const fn of Object.keys(allModuleExports[module])) { + if (allModuleExports[module][fn]?._called) { + const _fn = allModuleExports[module][fn]; + process.send({ + type: 'coverage', + module, + fn, + times: _fn._called, + }); + } + } + } +}); diff --git a/benchmark/run.js b/benchmark/run.js index 6a61df71221710..0e8c33dea24bb3 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -1,7 +1,8 @@ 'use strict'; -const path = require('path'); +const path = require('node:path'); const { spawn, fork } = require('node:child_process'); +const fs = require('node:fs'); const CLI = require('./_cli.js'); const cli = new CLI(`usage: ./node run.js [options] [--] ... @@ -16,6 +17,8 @@ const cli = new CLI(`usage: ./node run.js [options] [--] ... --format [simple|csv] optional value that specifies the output format test only run a single configuration from the options matrix + coverage generate a coverage report for the nodejs + benchmark suite all each benchmark category is run one after the other Examples: @@ -41,15 +44,47 @@ if (!validFormats.includes(format)) { return; } -if (format === 'csv') { +if (format === 'csv' && !cli.coverage) { console.log('"filename", "configuration", "rate", "time"'); } +function fetchModules() { + const dir = fs.readdirSync(path.join(__dirname, '../lib')); + const allModuleExports = {}; + for (const f of dir) { + if (f.endsWith('.js') && !f.startsWith('_')) { + const moduleName = `node:${f.slice(0, f.length - 3)}`; + const exports = require(moduleName); + allModuleExports[moduleName] = {}; + for (const fnKey of Object.keys(exports)) { + if (typeof exports[fnKey] === 'function' && !fnKey.startsWith('_')) { + allModuleExports[moduleName] = { + ...allModuleExports[moduleName], + [fnKey]: 0, + }; + } + } + } + } + return allModuleExports; +} + +let allModuleExports = {}; +if (cli.coverage) { + allModuleExports = fetchModules(); +} + (function recursive(i) { const filename = benchmarks[i]; const scriptPath = path.resolve(__dirname, filename); - const args = cli.test ? ['--test'] : cli.optional.set; + const args = cli.test ? ['--test'] : [...cli.optional.set]; + + let execArgv = []; + if (cli.coverage) { + execArgv = ['-r', path.join(__dirname, './coverage.js'), '--experimental-sqlite', '--no-warnings']; + } + const cpuCore = cli.getCpuCoreSetting(); let child; if (cpuCore !== null) { @@ -60,6 +95,7 @@ if (format === 'csv') { child = fork( scriptPath, args, + { execArgv }, ); } @@ -69,6 +105,15 @@ if (format === 'csv') { } child.on('message', (data) => { + if (cli.coverage) { + if (data.type === 'coverage') { + if (allModuleExports[data.module][data.fn] !== undefined) { + delete allModuleExports[data.module][data.fn]; + } + } + return; + } + if (data.type !== 'report') { return; } @@ -102,3 +147,44 @@ if (format === 'csv') { } }); })(0); + +const skippedFunctionClasses = [ + 'EventEmitter', + 'Worker', + 'ClientRequest', + 'Readable', + 'StringDecoder', + 'TLSSocket', + 'MessageChannel', +]; + +const skippedModules = [ + 'node:cluster', + 'node:trace_events', + 'node:stream/promises', +]; + +if (cli.coverage) { + process.on('beforeExit', () => { + for (const key in allModuleExports) { + if (skippedModules.includes(key)) continue; + const tableData = []; + for (const innerKey in allModuleExports[key]) { + if ( + allModuleExports[key][innerKey].toString().match(/^class/) || + skippedFunctionClasses.includes(innerKey) + ) { + continue; + } + + tableData.push({ + [key]: innerKey, + Values: allModuleExports[key][innerKey], + }); + } + if (tableData.length) { + console.table(tableData); + } + } + }); +} diff --git a/benchmark/url/url-searchparams-update.js b/benchmark/url/url-searchparams-update.js index 082d476a5d2250..3c42de61110ef3 100644 --- a/benchmark/url/url-searchparams-update.js +++ b/benchmark/url/url-searchparams-update.js @@ -17,7 +17,7 @@ function getMethod(url, property) { function main({ searchParams, property, n }) { const url = new URL('https://nodejs.org'); - if (searchParams === 'true') assert(url.searchParams); + if (searchParams === 'true') assert.ok(url.searchParams); const method = getMethod(url, property);