Add page render benchmark (#6415)

.github/workflows/benchmark.yml

@@ -95,7 +95,7 @@ jobs:
  continue-on-error: true
  with:
  issue-number: ${{ github.event.issue.number }}
- message: |
+ body: |
  ${{ needs.benchmark.outputs.PR-BENCH }}
 
  ${{ needs.benchmark.outputs.MAIN-BENCH }}

benchmark/bench/render.js

@@ -0,0 +1,122 @@
+import fs from 'fs/promises';
+import http from 'http';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { execaCommand } from 'execa';
+import { waitUntilBusy } from 'port-authority';
+import { markdownTable } from 'markdown-table';
+import { renderFiles } from '../make-project/render-default.js';
+import { astroBin } from './_util.js';
+
+const port = 4322;
+
+export const defaultProject = 'render-default';
+
+/** @typedef {{ avg: number, stdev: number, max: number }} Stat */
+
+/**
+ * @param {URL} projectDir
+ * @param {URL} outputFile
+ */
+export async function run(projectDir, outputFile) {
+ const root = fileURLToPath(projectDir);
+
+ console.log('Building...');
+ await execaCommand(`${astroBin} build`, {
+ cwd: root,
+ stdio: 'inherit',
+ });
+
+ console.log('Previewing...');
+ const previewProcess = execaCommand(`${astroBin} preview --port ${port}`, {
+ cwd: root,
+ stdio: 'inherit',
+ });
+
+ console.log('Waiting for server ready...');
+ await waitUntilBusy(port, { timeout: 5000 });
+
+ console.log('Running benchmark...');
+ const result = await benchmarkRenderTime();
+
+ console.log('Killing server...');
+ if (!previewProcess.kill('SIGTERM')) {
+ console.warn('Failed to kill server process id:', previewProcess.pid);
+ }
+
+ console.log('Writing results to', fileURLToPath(outputFile));
+ await fs.writeFile(outputFile, JSON.stringify(result, null, 2));
+
+ console.log('Result preview:');
+ console.log('='.repeat(10));
+ console.log(`#### Render\n\n`);
+ console.log(printResult(result));
+ console.log('='.repeat(10));
+
+ console.log('Done!');
+}
+
+async function benchmarkRenderTime() {
+ /** @type {Record<string, number[]>} */
+ const result = {};
+ for (const fileName of Object.keys(renderFiles)) {
+ // Render each file 100 times and push to an array
+ for (let i = 0; i < 100; i++) {
+ const pathname = '/' + fileName.slice(0, -path.extname(fileName).length);
+ const renderTime = await fetchRenderTime(`http:https://localhost:${port}${pathname}`);
+ if (!result[pathname]) result[pathname] = [];
+ result[pathname].push(renderTime);
+ }
+ }
+ /** @type {Record<string, Stat>} */
+ const processedResult = {};
+ for (const [pathname, times] of Object.entries(result)) {
+ // From the 100 results, calculate average, standard deviation, and max value
+ const avg = times.reduce((a, b) => a + b, 0) / times.length;
+ const stdev = Math.sqrt(
+ times.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / times.length
+ );
+ const max = Math.max(...times);
+ processedResult[pathname] = { avg, stdev, max };
+ }
+ return processedResult;
+}
+
+/**
+ * @param {Record<string, Stat>} result
+ */
+function printResult(result) {
+ return markdownTable(
+ [
+ ['Page', 'Avg (ms)', 'Stdev (ms)', 'Max (ms)'],
+ ...Object.entries(result).map(([pathname, { avg, stdev, max }]) => [
+ pathname,
+ avg.toFixed(2),
+ stdev.toFixed(2),
+ max.toFixed(2),
+ ]),
+ ],
+ {
+ align: ['l', 'r', 'r', 'r'],
+ }
+ );
+}
+
+/**
+ * Simple fetch utility to get the render time sent by `@astrojs/timer` in plain text
+ * @param {string} url
+ * @returns {Promise<number>}
+ */
+function fetchRenderTime(url) {
+ return new Promise((resolve, reject) => {
+ const req = http.request(url, (res) => {
+ res.setEncoding('utf8');
+ let data = '';
+ res.on('data', (chunk) => (data += chunk));
+ res.on('error', (e) => reject(e));
+ res.on('end', () => resolve(+data));
+ });
+ req.on('error', (e) => reject(e));
+ req.end();
+ });
+}

benchmark/index.js

@@ -12,6 +12,7 @@ astro-benchmark <command> [options]
 Command
  [empty] Run all benchmarks
  memory Run build memory and speed test
+ render Run rendering speed test
  server-stress Run server stress test
 
 Options
@@ -24,6 +25,7 @@ Options
 const commandName = args._[0];
 const benchmarks = {
  memory: () => import('./bench/memory.js'),
+ 'render': () => import('./bench/render.js'),
  'server-stress': () => import('./bench/server-stress.js'),
 };
 

benchmark/make-project/render-default.js

@@ -0,0 +1,87 @@
+import fs from 'fs/promises';
+import { loremIpsumHtml, loremIpsumMd } from './_util.js';
+
+// Map of files to be generated and tested for rendering.
+// Ideally each content should be similar for comparison.
+export const renderFiles = {
+ 'astro.astro': `\
+---
+const className = "text-red-500";
+const style = { color: "red" };
+const items = Array.from({ length: 1000 }, (_, i) => i);
+---
+
+<html>
+ <head>
+ <title>My Site</title>
+ </head>
+ <body>
+ <h1 class={className + ' text-lg'}>List</h1>
+ <ul style={style}>
+ {items.map((item) => (
+ <li class={className}>{item}</li>
+ ))}
+ </ul>
+ ${Array.from({ length: 1000 })
+ .map(() => `<p>${loremIpsumHtml}</p>`)
+ .join('\n')}
+ </body>
+</html>`,
+ 'md.md': `\
+# List
+
+${Array.from({ length: 1000 }, (_, i) => i)
+ .map((v) => `- ${v}`)
+ .join('\n')}
+
+${Array.from({ length: 1000 })
+ .map(() => loremIpsumMd)
+ .join('\n\n')}
+`,
+ 'mdx.mdx': `\
+export const className = "text-red-500";
+export const style = { color: "red" };
+export const items = Array.from({ length: 1000 }, (_, i) => i);
+
+# List
+
+<ul style={style}>
+ {items.map((item) => (
+ <li class={className}>{item}</li>
+ ))}
+</ul>
+
+${Array.from({ length: 1000 })
+ .map(() => loremIpsumMd)
+ .join('\n\n')}
+`,
+};
+
+/**
+ * @param {URL} projectDir
+ */
+export async function run(projectDir) {
+ await fs.rm(projectDir, { recursive: true, force: true });
+ await fs.mkdir(new URL('./src/pages', projectDir), { recursive: true });
+
+ await Promise.all(
+ Object.entries(renderFiles).map(([name, content]) => {
+ return fs.writeFile(new URL(`./src/pages/${name}`, projectDir), content, 'utf-8');
+ })
+ );
+
+ await fs.writeFile(
+ new URL('./astro.config.js', projectDir),
+ `\
+import { defineConfig } from 'astro/config';
+import timer from '@astrojs/timer';
+import mdx from '@astrojs/mdx';
+
+export default defineConfig({
+ integrations: [mdx()],
+ output: 'server',
+ adapter: timer(),
+});`,
+ 'utf-8'
+ );
+}

benchmark/package.json

@@ -7,7 +7,9 @@
  "astro-benchmark": "./index.js"
  },
  "dependencies": {
+ "@astrojs/mdx": "workspace:*",
  "@astrojs/node": "workspace:*",
+ "@astrojs/timer": "workspace:*",
  "astro": "workspace:*",
  "autocannon": "^7.10.0",
  "execa": "^6.1.0",

packages/integrations/timer/README.md

@@ -0,0 +1,3 @@
+# @astrojs/timer
+
+Like `@astrojs/node`, but returns the rendered time in milliseconds for the page instead of the page content itself. This is used for internal benchmarks only.

packages/integrations/timer/package.json

@@ -0,0 +1,43 @@
+{
+ "name": "@astrojs/timer",
+ "description": "Preview server for benchmark",
+ "private": true,
+ "version": "0.0.0",
+ "type": "module",
+ "types": "./dist/index.d.ts",
+ "author": "withastro",
+ "license": "MIT",
+ "repository": {
+ "type": "git",
+ "url": "https://github.com/withastro/astro.git",
+ "directory": "packages/integrations/timer"
+ },
+ "keywords": [
+ "withastro",
+ "astro-adapter"
+ ],
+ "bugs": "https://github.com/withastro/astro/issues",
+ "exports": {
+ ".": "./dist/index.js",
+ "./server.js": "./dist/server.js",
+ "./preview.js": "./dist/preview.js",
+ "./package.json": "./package.json"
+ },
+ "scripts": {
+ "build": "astro-scripts build \"src/**/*.ts\" && tsc",
+ "build:ci": "astro-scripts build \"src/**/*.ts\"",
+ "dev": "astro-scripts dev \"src/**/*.ts\""
+ },
+ "dependencies": {
+ "@astrojs/webapi": "workspace:*",
+ "server-destroy": "^1.0.1"
+ },
+ "peerDependencies": {
+ "astro": "workspace:^2.0.17"
+ },
+ "devDependencies": {
+ "@types/server-destroy": "^1.0.1",
+ "astro": "workspace:*",
+ "astro-scripts": "workspace:*"
+ }
+}

packages/integrations/timer/src/index.ts

@@ -0,0 +1,34 @@
+import type { AstroAdapter, AstroIntegration } from 'astro';
+
+export function getAdapter(): AstroAdapter {
+ return {
+ name: '@astrojs/timer',
+ serverEntrypoint: '@astrojs/timer/server.js',
+ previewEntrypoint: '@astrojs/timer/preview.js',
+ exports: ['handler'],
+ };
+}
+
+export default function createIntegration(): AstroIntegration {
+ return {
+ name: '@astrojs/timer',
+ hooks: {
+ 'astro:config:setup': ({ updateConfig }) => {
+ updateConfig({
+ vite: {
+ ssr: {
+ noExternal: ['@astrojs/timer'],
+ },
+ },
+ });
+ },
+ 'astro:config:done': ({ setAdapter, config }) => {
+ setAdapter(getAdapter());
+
+ if (config.output === 'static') {
+ console.warn(`[@astrojs/timer] \`output: "server"\` is required to use this adapter.`);
+ }
+ },
+ },
+ };
+}

packages/integrations/timer/src/preview.ts

@@ -0,0 +1,36 @@
+import type { CreatePreviewServer } from 'astro';
+import { createServer } from 'http';
+import enableDestroy from 'server-destroy';
+
+const preview: CreatePreviewServer = async function ({ serverEntrypoint, host, port }) {
+ const ssrModule = await import(serverEntrypoint.toString());
+ const ssrHandler = ssrModule.handler;
+ const server = createServer(ssrHandler);
+ server.listen(port, host);
+ enableDestroy(server);
+
+ // eslint-disable-next-line no-console
+ console.log(`Preview server listening on http:https://${host}:${port}`);
+
+ // Resolves once the server is closed
+ const closed = new Promise<void>((resolve, reject) => {
+ server.addListener('close', resolve);
+ server.addListener('error', reject);
+ });
+
+ return {
+ host,
+ port,
+ closed() {
+ return closed;
+ },
+ server,
+ stop: async () => {
+ await new Promise((resolve, reject) => {
+ server.destroy((err) => (err ? reject(err) : resolve(undefined)));
+ });
+ },
+ };
+};
+
+export { preview as default };