feat: add benchmarks for build times

.github/workflows/continuous_benchmark.yml

@@ -46,11 +46,15 @@ jobs:
       - name: Build
         run: pnpm run build
 
+      - name: Setup benchmark projects
+        working-directory: ./benchmark
+        run: pnpm run build:bench
+        timeout-minutes: 15
+
       - name: Run the benchmarks
         uses: CodSpeedHQ/action@76578c2a7ddd928664caa737f0e962e3085d4e7c # v3.8.1
         timeout-minutes: 30
         with:
           working-directory: ./benchmark
           run: pnpm bench
           token: ${{ secrets.CODSPEED_TOKEN }}
-

benchmark/bench/codspeed-setup.js

@@ -0,0 +1,27 @@
+import { fileURLToPath } from 'node:url';
+import { exec } from 'tinyexec';
+import { astroBin, makeProject } from './_util.js';
+
+/**
+ * Setup script for codspeed benchmarks.
+ * This builds the render-bench project which is required for the rendering benchmarks.
+ * This is separated out so it can run in a separate CI job since it takes a long time.
+ */
+async function setup() {
+	console.log('Setting up render-bench project...');
+	const render = await makeProject('render-bench');
+	const root = fileURLToPath(render);
+
+	console.log(`Building project at ${root}...`);
+	await exec(astroBin, ['build'], {
+		nodeOptions: {
+			cwd: root,
+			stdio: 'inherit',
+		},
+	});
+}
+
+setup().catch((error) => {
+	console.error('Setup failed:', error);
+	process.exit(1);
+});

benchmark/bench/codspeed.bench.js

@@ -1,20 +1,22 @@
+import { existsSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
-import { exec } from 'tinyexec';
+import { build } from 'astro';
 import { beforeAll, bench, describe } from 'vitest';
-import { astroBin, makeProject } from './_util.js';
 
+const staticRoot = fileURLToPath(new URL('../static-projects/build-static', import.meta.url));
+const hybridRoot = fileURLToPath(new URL('../static-projects/build-hybrid', import.meta.url));
+const serverRoot = fileURLToPath(new URL('../static-projects/build-server', import.meta.url));
+const renderRoot = new URL('../projects/render-bench/', import.meta.url);
+
 let streamingApp;
 let nonStreamingApp;
 beforeAll(async () => {
-	const render = await makeProject('render-bench');
-	const root = fileURLToPath(render);
-	await exec(astroBin, ['build'], {
-		nodeOptions: {
-			cwd: root,
-			stdio: 'inherit',
-		},
-	});
-	const entry = new URL('./dist/server/entry.mjs', `file://${root}`);
+	const entry = new URL('./dist/server/entry.mjs', renderRoot);
+
+	if (!existsSync(fileURLToPath(entry))) {
+		throw new Error('render-bench project not built. Please run `pnpm run build:bench` before running the benchmarks.');
+	}
+
 	const { manifest, createApp } = await import(entry);
 	streamingApp = createApp(manifest, true);
 	nonStreamingApp = createApp(manifest, false);
@@ -47,3 +49,26 @@ describe('Bench rendering', () => {
 		await nonStreamingApp.render(request);
 	});
 });
+
+describe('Bench build time', () => {
+	bench('Build: full static site', async () => {
+		await build({
+			root: staticRoot,
+			logLevel: 'error',
+		});
+	}, { timeout: 300000, iterations: 3 });
+
+	bench('Build: hybrid site (static + server)', async () => {
+		await build({
+			root: hybridRoot,
+			logLevel: 'error',
+		});
+	}, { timeout: 300000, iterations: 3 });
+
+	bench('Build: full server site', async () => {
+		await build({
+			root: serverRoot,
+			logLevel: 'error',
+		});
+	}, { timeout: 300000, iterations: 3 });
+});

benchmark/package.json

@@ -7,7 +7,8 @@
     "astro-benchmark": "./index.js"
   },
   "scripts": {
-    "bench": "pnpm vitest bench --run"
+    "bench": "pnpm vitest bench --run",
+    "build:bench": "node ./bench/codspeed-setup.js"
   },
   "dependencies": {
     "@astrojs/mdx": "workspace:*",
@@ -24,7 +25,7 @@
     "tinyexec": "^1.0.2"
   },
   "devDependencies": {
-    "@codspeed/vitest-plugin": "4.0.1",
+    "@codspeed/vitest-plugin": "5.0.1",
     "vitest": "^3.2.4"
   }
 }

benchmark/static-projects/build-hybrid/astro.config.js

@@ -0,0 +1,8 @@
+import { defineConfig } from 'astro/config';
+import node from '@astrojs/node';
+
+export default defineConfig({
+	adapter: node({
+    mode: 'standalone',
+  }),
+});

benchmark/static-projects/build-hybrid/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "benchmark-build-hybrid",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@astrojs/node": "workspace:^",
+    "astro": "workspace:^"
+  }
+}

benchmark/static-projects/build-hybrid/src/components/Card.astro

@@ -0,0 +1,8 @@
+---
+const { title, description, href } = Astro.props;
+---
+<article class="card">
+	<h3>{title}</h3>
+	<p>{description}</p>
+	<a href={href}>Read more</a>
+</article>

benchmark/static-projects/build-hybrid/src/layouts/BaseLayout.astro

@@ -0,0 +1,28 @@
+---
+const { title } = Astro.props;
+---
+
+<!doctype html>
+<html lang="en">
+	<head>
+		<meta charset="UTF-8" />
+		<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+		<title>{title}</title>
+	</head>
+	<body>
+		<header>
+			<nav>
+				<a href="/">Home</a>
+				<a href="/about">About</a>
+				<a href="/blog">Blog</a>
+				<a href="/api/data.json">API</a>
+			</nav>
+		</header>
+		<main>
+			<slot />
+		</main>
+		<footer>
+			<p>&copy; 2024 Hybrid Site</p>
+		</footer>
+	</body>
+</html>

benchmark/static-projects/build-hybrid/src/pages/about.astro

@@ -0,0 +1,18 @@
+---
+import BaseLayout from '../layouts/BaseLayout.astro';
+
+export const prerender = true;
+---
+<BaseLayout title="About - Hybrid Site">
+	<h1>About Us</h1>
+	<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+	<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+	<section>
+		<h2>Our Team</h2>
+		<ul>
+			{Array.from({ length: 20 }, (_, i) => (
+				<li>Team Member {i + 1}</li>
+			))}
+		</ul>
+	</section>
+</BaseLayout>

benchmark/static-projects/build-hybrid/src/pages/api/data.json.js

@@ -0,0 +1,16 @@
+export const prerender = false;
+
+export async function GET({ params, request }) {
+	return new Response(JSON.stringify({
+		timestamp: new Date().toISOString(),
+		data: Array.from({ length: 100 }, (_, i) => ({
+			id: i + 1,
+			title: `Item ${i + 1}`,
+		})),
+	}), {
+		status: 200,
+		headers: {
+			'Content-Type': 'application/json'
+		}
+	});
+}

benchmark/static-projects/build-hybrid/src/pages/blog/[id].astro

@@ -0,0 +1,36 @@
+---
+import BaseLayout from '../../layouts/BaseLayout.astro';
+
+export async function getStaticPaths() {
+	// Only pre-render posts 1-50
+	return Array.from({ length: 50 }, (_, i) => ({
+		params: { id: String(i + 1) },
+		props: { postNumber: i + 1, isStatic: true },
+	}));
+}
+
+const { postNumber, isStatic } = Astro.props;
+const id = Astro.params.id;
+const currentPostNumber = postNumber || Number(id);
+const timestamp = new Date().toISOString();
+---
+<BaseLayout title={`Blog Post ${currentPostNumber} - Hybrid Site`}>
+	<article>
+		<h1>Blog Post {currentPostNumber}{!isStatic && ' (Server-Rendered)'}</h1>
+		<p class="meta">Published on January {(currentPostNumber % 28) + 1}, 2024</p>
+		{!isStatic && <p class="server-time">Server time: {timestamp}</p>}
+		<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+		<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+		<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+		<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+		<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+		<section>
+			<h2>Section 1</h2>
+			<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+		</section>
+		<section>
+			<h2>Section 2</h2>
+			<p><strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>. <strong>Lorem</strong> <em>Ipsum</em> is simply <span>dummy</span> text of the printing and typesetting industry. <strong>Lorem</strong> <em>Ipsum</em> has been the industry's standard <span>dummy</span> text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing <strong>Lorem</strong> <em>Ipsum</em> passages, and more recently with desktop publishing software like Aldus PageMaker including versions of <strong>Lorem</strong> <em>Ipsum</em>.</p>
+		</section>
+	</article>
+</BaseLayout>

benchmark/static-projects/build-hybrid/src/pages/blog/index.astro

@@ -0,0 +1,23 @@
+---
+import BaseLayout from '../../layouts/BaseLayout.astro';
+import Card from '../../components/Card.astro';
+
+export const prerender = true;
+
+const posts = Array.from({ length: 100 }, (_, i) => ({
+	title: `Blog Post ${i + 1}`,
+	description: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit.',
+	href: `/blog/${i + 1}`,
+}));
+---
+
+<BaseLayout title="Blog - Hybrid Site">
+	<h1>All Blog Posts</h1>
+	<div class="grid">
+		{
+			posts.map((post) => (
+				<Card title={post.title} description={post.description} href={post.href} />
+			))
+		}
+	</div>
+</BaseLayout>