Add collections to build

src/build.ts

@@ -1,6 +1,6 @@
 import type { AstroConfig, RuntimeMode } from './@types/astro';
 import type { LogOptions } from './logger';
-import type { LoadResult } from './runtime';
+import type { AstroRuntime, LoadResult } from './runtime';
 
 import { existsSync, promises as fsPromises } from 'fs';
 import { relative as pathRelative } from 'path';
@@ -13,6 +13,18 @@ import { collectStatics } from './build/static.js';
 
 const { mkdir, readdir, readFile, stat, writeFile } = fsPromises;
 
+interface PageBuildOptions {
+ astroRoot: URL;
+ dist: URL;
+ filepath: URL;
+ runtime: AstroRuntime;
+ statics: Set<string>;
+}
+
+interface PageResult {
+ statusCode: number;
+}
+
 const logging: LogOptions = {
  level: 'debug',
  dest: defaultLogDestination,
@@ -55,6 +67,78 @@ async function writeResult(result: LoadResult, outPath: URL, encoding: null | 'u
  }
 }
 
+/** Collection utility */
+function getPageType(filepath: URL): 'collection' | 'static' {
+ if (/\$[^.]+.astro$/.test(filepath.pathname)) return 'collection';
+ return 'static';
+}
+
+/** Build collection */
+async function buildCollectionPage({ astroRoot, dist, filepath, runtime, statics }: PageBuildOptions): Promise<PageResult> {
+ const rel = pathRelative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro
+ const pagePath = `/${rel.replace(/\$([^.]+)\.astro$/, '$1')}`;
+ const builtURLs = new Set<string>(); // !important: internal cache that prevents building the same URLs
+
+ /** Recursively build collection URLs */
+ async function loadCollection(url: string): Promise<LoadResult | undefined> {
+ if (builtURLs.has(url)) return; // this stops us from recursively building the same pages over and over
+ const result = await runtime.load(url);
+ builtURLs.add(url);
+ if (result.statusCode === 200) {
+ const outPath = new URL('./' + url + '/index.html', dist);
+ await writeResult(result, outPath, 'utf-8');
+ mergeSet(statics, collectStatics(result.contents.toString('utf-8')));
+ }
+ return result;
+ }
+
+ const result = (await loadCollection(pagePath)) as LoadResult;
+ if (result.statusCode === 200 && !result.collectionInfo) {
+ throw new Error(`[${rel}]: Collection page must export createCollection() function`);
+ }
+
+ // note: for pages that require params (/tag/:tag), we will get a 404 but will still get back collectionInfo that tell us what the URLs should be
+ if (result.collectionInfo) {
+ await Promise.all(
+ [...result.collectionInfo.additionalURLs].map(async (url) => {
+ // for the top set of additional URLs, we render every new URL generated
+ const addlResult = await loadCollection(url);
+ if (addlResult && addlResult.collectionInfo) {
+ // believe it or not, we may still have a few unbuilt pages left. this is our last crawl:
+ await Promise.all([...addlResult.collectionInfo.additionalURLs].map(async (url2) => loadCollection(url2)));
+ }
+ })
+ );
+ }
+
+ return {
+ statusCode: result.statusCode,
+ };
+}
+
+/** Build static page */
+async function buildStaticPage({ astroRoot, dist, filepath, runtime, statics }: PageBuildOptions): Promise<PageResult> {
+ const rel = pathRelative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro
+ const pagePath = `/${rel.replace(/\.(astro|md)$/, '')}`;
+
+ let relPath = './' + rel.replace(/\.(astro|md)$/, '.html');
+ if (!relPath.endsWith('index.html')) {
+ relPath = relPath.replace(/\.html$/, '/index.html');
+ }
+
+ const outPath = new URL(relPath, dist);
+ const result = await runtime.load(pagePath);
+
+ await writeResult(result, outPath, 'utf-8');
+ if (result.statusCode === 200) {
+ mergeSet(statics, collectStatics(result.contents.toString('utf-8')));
+ }
+
+ return {
+ statusCode: result.statusCode,
+ };
+}
+
 /** The primary build action */
 export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
  const { projectRoot, astroRoot } = astroConfig;
@@ -77,30 +161,27 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
  const statics = new Set<string>();
  const collectImportsOptions = { astroConfig, logging, resolve, mode };
 
- for (const pathname of await allPages(pageRoot)) {
- const filepath = new URL(`file:https://${pathname}`);
- const rel = pathRelative(astroRoot.pathname + '/pages', filepath.pathname); // pages/index.astro
- const pagePath = `/${rel.replace(/\.(astro|md)/, '')}`;
-
- try {
- let relPath = './' + rel.replace(/\.(astro|md)$/, '.html');
- if (!relPath.endsWith('index.html')) {
- relPath = relPath.replace(/\.html$/, '/index.html');
- }
-
- const outPath = new URL(relPath, dist);
- const result = await runtime.load(pagePath);
-
- await writeResult(result, outPath, 'utf-8');
- if (result.statusCode === 200) {
- mergeSet(statics, collectStatics(result.contents.toString('utf-8')));
- }
- } catch (err) {
- error(logging, 'generate', err);
- return 1;
- }
-
- mergeSet(imports, await collectDynamicImports(filepath, collectImportsOptions));
+ const pages = await allPages(pageRoot);
+
+ try {
+ await Promise.all(
+ pages.map(async (pathname) => {
+ const filepath = new URL(`file:https://${pathname}`);
+
+ const pageType = getPageType(filepath);
+ const pageOptions: PageBuildOptions = { astroRoot, dist, filepath, runtime, statics };
+ if (pageType === 'collection') {
+ await buildCollectionPage(pageOptions);
+ } else {
+ await buildStaticPage(pageOptions);
+ }
+
+ mergeSet(imports, await collectDynamicImports(filepath, collectImportsOptions));
+ })
+ );
+ } catch (err) {
+ error(logging, 'generate', err);
+ return 1;
  }
 
  for (const pathname of await allPages(componentRoot)) {

src/runtime.ts

@@ -25,16 +25,19 @@ interface RuntimeConfig {
  frontendSnowpackConfig: SnowpackConfig;
 }
 
+// info needed for collection generation
+type CollectionInfo = { additionalURLs: Set<string> };
+
 type LoadResultSuccess = {
  statusCode: 200;
  contents: string | Buffer;
  contentType?: string | false;
 };
-type LoadResultNotFound = { statusCode: 404; error: Error };
-type LoadResultRedirect = { statusCode: 301 | 302; location: string };
+type LoadResultNotFound = { statusCode: 404; error: Error; collectionInfo?: CollectionInfo };
+type LoadResultRedirect = { statusCode: 301 | 302; location: string; collectionInfo?: CollectionInfo };
 type LoadResultError = { statusCode: 500 } & ({ type: 'parse-error'; error: CompileError } | { type: 'unknown'; error: Error });
 
-export type LoadResult = LoadResultSuccess | LoadResultNotFound | LoadResultRedirect | LoadResultError;
+export type LoadResult = (LoadResultSuccess | LoadResultNotFound | LoadResultRedirect | LoadResultError) & { collectionInfo?: CollectionInfo };
 
 // Disable snowpack from writing to stdout/err.
 snowpackLogger.level = 'silent';
@@ -82,6 +85,8 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
 
  // handle collection
  let collection = {} as CollectionResult;
+ let additionalURLs = new Set<string>();
+
  if (mod.exports.createCollection) {
  const createCollection: CreateCollection = await mod.exports.createCollection();
  for (const key of Object.keys(createCollection)) {
@@ -100,6 +105,7 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
  }
  let requestedParams = routes.find((p) => {
  const baseURL = (permalink as any)({ params: p });
+ additionalURLs.add(baseURL);
  return baseURL === reqPath || `${baseURL}/${searchResult.currentPage || 1}` === reqPath;
  });
  if (requestedParams) {
@@ -135,22 +141,38 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
  .replace(/\/1$/, ''); // if end is `/1`, then just omit
  }
 
+ // from page 2 to the end, add all pages as additional URLs (needed for build)
+ for (let n = 1; n <= collection.page.last; n++) {
+ if (additionalURLs.size) {
+ // if this is a param-based collection, paginate all params
+ additionalURLs.forEach((url) => {
+ additionalURLs.add(url.replace(/(\/\d+)?$/, `/${n}`));
+ });
+ } else {
+ // if this has no params, simply add page
+ additionalURLs.add(reqPath.replace(/(\/\d+)?$/, `/${n}`));
+ }
+ }
+
  data = data.slice(start, end);
  } else if (createCollection.pageSize) {
  // TODO: fix bug where redirect doesn’t happen
  // This happens because a pageSize is set, but the user isn’t on a paginated route. Redirect:
  return {
  statusCode: 301,
  location: reqPath + '/1',
+ collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
  };
  }
 
  // if we’ve paginated too far, this is a 404
- if (!data.length)
+ if (!data.length) {
  return {
  statusCode: 404,
  error: new Error('Not Found'),
+ collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
  };
+ }
 
  collection.data = data;
  }
@@ -178,6 +200,7 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
  statusCode: 200,
  contentType: 'text/html; charset=utf-8',
  contents: html,
+ collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
  };
  } catch (err) {
  if (err.code === 'parse-error') {