Add CLI startup speed benchmark (#7617)

benchmark/bench/_util.js

@@ -1,3 +1,18 @@
 import { createRequire } from 'module';
 
 export const astroBin = createRequire(import.meta.url).resolve('astro');
+
+/** @typedef {{ avg: number, stdev: number, max: number }} Stat */
+
+/**
+ * @param {number[]} numbers
+ * @returns {Stat}
+ */
+export function calculateStat(numbers) {
+ const avg = numbers.reduce((a, b) => a + b, 0) / numbers.length;
+ const stdev = Math.sqrt(
+ numbers.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / numbers.length
+ );
+ const max = Math.max(...numbers);
+ return { avg, stdev, max };
+}

benchmark/bench/cli-startup.js

@@ -0,0 +1,73 @@
+import { fileURLToPath } from 'url';
+import { execaCommand } from 'execa';
+import { markdownTable } from 'markdown-table';
+import { astroBin, calculateStat } from './_util.js';
+
+/** Default project to run for this benchmark if not specified */
+export const defaultProject = 'render-default';
+
+/**
+ * @param {URL} projectDir
+ * @param {URL} outputFile
+ */
+export async function run(projectDir, outputFile) {
+ const root = fileURLToPath(projectDir);
+
+ console.log('Benchmarking `astro --help`...');
+ const helpStat = await benchmarkCommand(`node ${astroBin} --help`, root);
+ console.log('Done');
+
+ console.log('Benchmarking `astro info`...');
+ const infoStat = await benchmarkCommand(`node ${astroBin} info`, root);
+ console.log('Done');
+
+ console.log('Result preview:');
+ console.log('='.repeat(10));
+ console.log(`#### CLI Startup\n\n`);
+ console.log(
+ printResult({
+ 'astro --help': helpStat,
+ 'astro info': infoStat,
+ })
+ );
+ console.log('='.repeat(10));
+}
+
+/**
+ * @param {string} command
+ * @param {string} root
+ * @returns {Promise<import('./_util.js').Stat>}
+ */
+async function benchmarkCommand(command, root) {
+ /** @type {number[]} */
+ const durations = [];
+
+ for (let i = 0; i < 10; i++) {
+ const start = performance.now();
+ await execaCommand(command, { cwd: root });
+ durations.push(performance.now() - start);
+ }
+
+ // From the 10 durations, calculate average, standard deviation, and max value
+ return calculateStat(durations);
+}
+
+/**
+ * @param {Record<string, import('./_util.js').Stat>} result
+ */
+function printResult(result) {
+ return markdownTable(
+ [
+ ['Command', 'Avg (ms)', 'Stdev (ms)', 'Max (ms)'],
+ ...Object.entries(result).map(([command, { avg, stdev, max }]) => [
+ command,
+ avg.toFixed(2),
+ stdev.toFixed(2),
+ max.toFixed(2),
+ ]),
+ ],
+ {
+ align: ['l', 'r', 'r', 'r'],
+ }
+ );
+}

benchmark/bench/render.js

@@ -6,14 +6,13 @@ import { execaCommand } from 'execa';
 import { waitUntilBusy } from 'port-authority';
 import { markdownTable } from 'markdown-table';
 import { renderFiles } from '../make-project/render-default.js';
+import { calculateStat } from '../make-project/_util.js';
 import { astroBin } from './_util.js';
 
 const port = 4322;
 
 export const defaultProject = 'render-default';
 
-/** @typedef {{ avg: number, stdev: number, max: number }} Stat */
-
 /**
  * @param {URL} projectDir
  * @param {URL} outputFile
@@ -68,22 +67,17 @@ async function benchmarkRenderTime() {
  result[pathname].push(renderTime);
  }
  }
- /** @type {Record<string, Stat>} */
+ /** @type {Record<string, import('./_util.js').Stat>} */
  const processedResult = {};
  for (const [pathname, times] of Object.entries(result)) {
  // From the 100 results, calculate average, standard deviation, and max value
- const avg = times.reduce((a, b) => a + b, 0) / times.length;
- const stdev = Math.sqrt(
- times.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / times.length
- );
- const max = Math.max(...times);
- processedResult[pathname] = { avg, stdev, max };
+ processedResult[pathname] = calculateStat(times);
  }
  return processedResult;
 }
 
 /**
- * @param {Record<string, Stat>} result
+ * @param {Record<string, import('./_util.js').Stat>} result
  */
 function printResult(result) {
  return markdownTable(

benchmark/index.js

@@ -14,6 +14,7 @@ Command
  memory Run build memory and speed test
  render Run rendering speed test
  server-stress Run server stress test
+ cli-startup Run CLI startup speed test
 
 Options
  --project <project-name> Project to use for benchmark, see benchmark/make-project/ for available names
@@ -27,6 +28,7 @@ const benchmarks = {
  memory: () => import('./bench/memory.js'),
  render: () => import('./bench/render.js'),
  'server-stress': () => import('./bench/server-stress.js'),
+ 'cli-startup': () => import('./bench/cli-startup.js'),
 };
 
 if (commandName && !(commandName in benchmarks)) {

packages/astro/src/cli/index.ts

@@ -113,7 +113,7 @@ async function printInfo({
  } catch (_e) {}
  console.log();
  printRow('Astro version', `v${ASTRO_VERSION}`);
- printRow('Package manager', packageManager.name);
+ printRow('Package manager', packageManager?.name);
  printRow('Platform', platform());
  printRow('Architecture', arch());
  printRow('Adapter', adapter);