Adding E2E testing with Playwright (withastro#3349)

* adding Tailwind E2E tests with Playwright

* package.json updates

* adding e2e tests to CI workflow

* using e2e for dev tests, mocha for build tests

* refactor: sharing test-utils helpers

* chore: update lockfile

* Adding contributing docs

* Revert "refactor: sharing test-utils helpers"

This reverts commit 48496f4.

* refactor: simpler solution to resolving e2e test fixtures

* chore: updating lockfile

* refactor: cleaning up how URLs are resolved in e2e tests

* install playwright deps in CI

* trying pnpm playwright install to fix version mismatch

.github/workflows/ci.yml

@@ -153,6 +153,48 @@ jobs:
  - name: Test
  run: pnpm run test
 
+ e2e:
+ name: 'E2E: ${{ matrix.os }} (node@${{ matrix.node_version }})'
+ runs-on: ${{ matrix.os }}
+ env:
+ ASTRO_TELEMETRY_DISABLED: true
+ strategy:
+ matrix:
+ os: [ubuntu-latest]
+ node_version: [14, 16]
+ include:
+ - os: windows-latest
+ node_version: 16
+ - os: macos-latest
+ node_version: 16
+ fail-fast: false
+ needs:
+ - build
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - name: Setup PNPM
+ uses: pnpm/[email protected]
+
+ - name: Setup node@${{ matrix.node_version }}
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node_version }}
+ cache: 'pnpm'
+
+ - name: Download Build Artifacts
+ uses: actions/download-artifact@v3
+
+ - name: Extract Artifacts
+ run: ./.github/extract-artifacts.sh
+
+ - name: Install dependencies
+ run: pnpm install
+
+ - name: Test
+ run: pnpm run test:e2e
+
  smoke:
  name: 'Test (Smoke) ${{ matrix.os }}'
  runs-on: ${{ matrix.os }}

CONTRIBUTING.md

@@ -59,6 +59,24 @@ pnpm run test
 pnpm run test:match "$STRING_MATCH"
 ```
 
+#### E2E tests
+
+Certain features, like HMR and client hydration, need end-to-end tests to verify functionality in the dev server. [Playwright](https://playwright.dev/) is used to test against the dev server.
+
+```shell
+# run this in the top-level project root to run all E2E tests
+pnpm run test:e2e
+# run only a few tests, great for working on a single feature
+# (example - `pnpm run test:e2e:match "Tailwind CSS" runs `tailwindcss.test.js`)
+pnpm run test:e2e:match "$STRING_MATCH"
+```
+
+**When should you add E2E tests?**
+
+Any tests for `astro build` output should use the main `mocha` tests rather than E2E - these tests will run faster than having Playwright start the `astro preview` server.
+
+If a test needs to validate what happens on the page after it's loading in the browser, that's a perfect use for E2E dev server tests, i.e. to verify that hot-module reloading works in `astro dev` or that components were client hydrated and are interactive.
+
 ### Other useful commands
 
 ```shell

package.json

@@ -18,6 +18,7 @@
  "test:templates": "turbo run test --filter=create-astro --concurrency=1",
  "test:smoke": "node scripts/smoke/index.js",
  "test:vite-ci": "turbo run test --no-deps --scope=astro --concurrency=1",
+ "test:e2e": "cd packages/astro && pnpm playwright install && pnpm run test:e2e",
  "benchmark": "turbo run benchmark --scope=astro",
  "lint": "eslint \"packages/**/*.ts\"",
  "format": "prettier -w .",

packages/astro/e2e/fixtures/tailwindcss/astro.config.mjs

@@ -0,0 +1,12 @@
+import { defineConfig } from 'astro/config';
+import tailwind from '@astrojs/tailwind';
+
+// https://astro.build/config
+export default defineConfig({
+ integrations: [tailwind()],
+ vite: {
+ build: {
+ assetsInlineLimit: 0,
+ },
+ },
+});

packages/astro/e2e/fixtures/tailwindcss/package.json

@@ -0,0 +1,9 @@
+{
+ "name": "@test/e2e-tailwindcss",
+ "version": "0.0.0",
+ "private": true,
+ "dependencies": {
+ "astro": "workspace:*",
+ "@astrojs/tailwind": "workspace:*"
+ }
+}

packages/astro/e2e/fixtures/tailwindcss/postcss.config.js

@@ -0,0 +1,9 @@
+const path = require('path');
+module.exports = {
+ plugins: {
+ tailwindcss: {
+ config: path.join(__dirname, 'tailwind.config.js'), // update this if your path differs!
+ },
+ autoprefixer: {}
+ },
+};

packages/astro/e2e/fixtures/tailwindcss/src/components/Button.astro

@@ -0,0 +1,10 @@
+---
+let { type = 'button' } = Astro.props;
+---
+
+<button
+ class="py-2 px-4 lg:py-3 lg:px-5 bg-purple-600 text-white font-[900] rounded-lg shadow-md hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-400 focus:ring-opacity-75"
+ {type}
+>
+ <slot />
+</button>

packages/astro/e2e/fixtures/tailwindcss/src/components/Complex.astro

@@ -0,0 +1 @@
+<div id="complex" class="w-10/12 2xl:w-[80%]"></div>

packages/astro/e2e/fixtures/tailwindcss/src/pages/index.astro

@@ -0,0 +1,18 @@
+---
+// Component Imports
+import Button from '../components/Button.astro';
+import Complex from '../components/Complex.astro';
+---
+
+<html lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <link rel="icon" type="image/x-icon" href="/favicon.ico" />
+ <title>Astro + TailwindCSS</title>
+ </head>
+
+ <body class="bg-dawn text-midnight">
+ <Button>I’m a Tailwind Button!</Button>
+ <Complex />
+ </body>
+</html>

packages/astro/e2e/fixtures/tailwindcss/src/pages/markdown-page.md

@@ -0,0 +1,11 @@
+---
+title: "Markdown + Tailwind"
+setup: |
+ import Button from '../components/Button.astro';
+ import Complex from '../components/Complex.astro';
+---
+
+<div class="grid place-items-center h-screen content-center">
+ <Button>Tailwind Button in Markdown!</Button>
+ <Complex />
+</div>

packages/astro/e2e/fixtures/tailwindcss/tailwind.config.js

@@ -0,0 +1,14 @@
+const path = require('path');
+
+module.exports = {
+ content: [path.join(__dirname, 'src/**/*.{astro,html,js,jsx,md,svelte,ts,tsx,vue}')],
+ theme: {
+ extend: {
+ colors: {
+ dawn: '#f3e9fa',
+ dusk: '#514375',
+ midnight: '#31274a',
+ }
+ }
+ }
+};

packages/astro/e2e/tailwindcss.test.js

@@ -0,0 +1,51 @@
+import { test as base, expect } from '@playwright/test';
+import { loadFixture } from './test-utils.js';
+
+const test = base.extend({
+ astro: async ({}, use) => {
+ const fixture = await loadFixture({ root: './fixtures/tailwindcss/' });
+ await use(fixture);
+ },
+});
+
+let devServer;
+
+test.beforeAll(async ({ astro }) => {
+ devServer = await astro.startDevServer();
+});
+
+test.afterAll(async ({ astro }) => {
+ await devServer.stop();
+});
+
+test('Tailwind CSS', async ({ page, astro }) => {
+ await page.goto(astro.resolveUrl('/'));
+
+ await test.step('body', async () => {
+ const body = page.locator('body');
+
+ await expect(body, 'should have classes').toHaveClass('bg-dawn text-midnight');
+ await expect(body, 'should have background color').toHaveCSS(
+ 'background-color',
+ 'rgb(243, 233, 250)'
+ );
+ await expect(body, 'should have color').toHaveCSS('color', 'rgb(49, 39, 74)');
+ });
+
+ await test.step('button', async () => {
+ const button = page.locator('button');
+
+ await expect(button, 'should have bg-purple-600').toHaveClass(/bg-purple-600/);
+ await expect(button, 'should have background color').toHaveCSS(
+ 'background-color',
+ 'rgb(147, 51, 234)'
+ );
+
+ await expect(button, 'should have lg:py-3').toHaveClass(/lg:py-3/);
+ await expect(button, 'should have padding bottom').toHaveCSS('padding-bottom', '12px');
+ await expect(button, 'should have padding top').toHaveCSS('padding-top', '12px');
+
+ await expect(button, 'should have font-[900]').toHaveClass(/font-\[900\]/);
+ await expect(button, 'should have font weight').toHaveCSS('font-weight', '900');
+ });
+});

packages/astro/e2e/test-utils.js

@@ -0,0 +1,13 @@
+import { loadFixture as baseLoadFixture } from '../test/test-utils.js';
+
+export function loadFixture(inlineConfig) {
+ if (!inlineConfig || !inlineConfig.root)
+ throw new Error("Must provide { root: './fixtures/...' }");
+
+ // resolve the relative root (i.e. "./fixtures/tailwindcss") to a full filepath
+ // without this, the main `loadFixture` helper will resolve relative to `packages/astro/test`
+ return baseLoadFixture({
+ ...inlineConfig,
+ root: new URL(inlineConfig.root, import.meta.url).toString()
+ })
+}

packages/astro/package.json

@@ -72,7 +72,8 @@
  "postbuild": "astro-scripts copy \"src/**/*.astro\"",
  "benchmark": "node test/benchmark/dev.bench.js && node test/benchmark/build.bench.js",
  "test": "mocha --exit --timeout 20000 --ignore **/lit-element.test.js && mocha --timeout 20000 **/lit-element.test.js",
- "test:match": "mocha --timeout 20000 -g"
+ "test:match": "mocha --timeout 20000 -g",
+ "test:e2e": "playwright test e2e"
  },
  "dependencies": {
  "@astrojs/compiler": "^0.14.2",
@@ -134,7 +135,8 @@
  "zod": "^3.16.0"
  },
  "devDependencies": {
- "@babel/types": "^7.17.10",
+ "@babel/types": "^7.17.0",
+ "@playwright/test": "^1.21.1",
  "@types/babel__core": "^7.1.19",
  "@types/babel__generator": "^7.6.4",
  "@types/babel__traverse": "^7.17.1",

packages/astro/test/tailwindcss.test.js

@@ -70,50 +70,4 @@ describe('Tailwind', () => {
  expect(bundledCSS, 'includes used component classes').to.match(/\.bg-purple-600{/);
  });
  });
-
- // with "build" handling CSS checking, the dev tests are mostly testing the paths resolve in dev
- describe('dev', () => {
- let devServer;
- let $;
-
- before(async () => {
- devServer = await fixture.startDevServer();
- const html = await fixture.fetch('/').then((res) => res.text());
- $ = cheerio.load(html);
- });
-
- after(async () => {
- devServer && (await devServer.stop());
- });
-
- it('resolves CSS in src/styles', async () => {
- const bundledCSSHREF = $('link[rel=stylesheet]').attr('href');
- const res = await fixture.fetch(bundledCSSHREF);
- expect(res.status).to.equal(200);
-
- const text = await res.text();
- expect(text, 'includes used component classes').to.match(/\.bg-purple-600/);
-
- // tests a random tailwind class that isn't used on the page
- expect(text, 'purges unused classes').not.to.match(/\.bg-blue-600/);
-
- // tailwind escapes colons, `lg:py-3` compiles to `lg\:py-3`
- expect(text, 'includes responsive classes').to.match(/\.lg\\\\:py-3/);
-
- // tailwind escapes brackets, `font-[900]` compiles to `font-\[900\]`
- expect(text, 'supports arbitrary value classes').to.match(/.font-\\[900\\]/);
-
- // custom theme colors were included
- expect(text, 'includes custom theme colors').to.match(/\.text-midnight/);
- expect(text, 'includes custom theme colors').to.match(/\.bg-dawn/);
- });
-
- it('maintains classes in HTML', async () => {
- const button = $('button');
-
- expect(button.hasClass('text-white'), 'basic class').to.be.true;
- expect(button.hasClass('lg:py-3'), 'responsive class').to.be.true;
- expect(button.hasClass('font-[900]', 'arbitrary value')).to.be.true;
- });
- });
 });

packages/astro/test/test-utils.js

@@ -25,6 +25,7 @@ polyfill(globalThis, {
  *
  * @typedef {Object} Fixture
  * @property {typeof build} build
+ * @property {(url: string) => string} resolveUrl
  * @property {(url: string, opts: any) => Promise<Response>} fetch
  * @property {(path: string) => Promise<string>} readFile
  * @property {(path: string) => Promise<string[]>} readdir
@@ -93,6 +94,8 @@ export async function loadFixture(inlineConfig) {
  },
  };
 
+ const resolveUrl = (url) => `http:https://${'127.0.0.1'}:${config.server.port}${url.replace(/^\/?/, '/')}`;
+
  return {
  build: (opts = {}) => build(config, { mode: 'development', logging, telemetry, ...opts }),
  startDevServer: async (opts = {}) => {
@@ -101,8 +104,9 @@ export async function loadFixture(inlineConfig) {
  return devResult;
  },
  config,
+ resolveUrl,
  fetch: (url, init) =>
- fetch(`http:https://${'127.0.0.1'}:${config.server.port}${url.replace(/^\/?/, '/')}`, init),
+ fetch(resolveUrl(url), init),
  preview: async (opts = {}) => {
  const previewServer = await preview(config, { logging, telemetry, ...opts });
  return previewServer;