Skip to content

Commit

Permalink
CI Reporter Infrastructure and expandedJson reports (#111)
Browse files Browse the repository at this point in the history
Co-authored-by: Tomas Jansson <[email protected]>
  • Loading branch information
harlan-zw and mastoj committed May 20, 2023
1 parent 01017d1 commit 1cf833e
Show file tree
Hide file tree
Showing 14 changed files with 36,712 additions and 84 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,6 @@ dist


reports

.vscode/
.tmp/
65 changes: 44 additions & 21 deletions docs/content/2.integrations/1.ci.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,28 @@ npm install -g @unlighthouse/cli puppeteer

## Usage

### Reporting

You may want to generate reports that can be consumed by other tools. To do you can use the reporter options:
- `jsonSimple` - A simple JSON report which contains the URL and score. This is the default.
- `jsonExpanded` - A full JSON report which contains the URL, score, metric and category breakdowns.
- `false` - Don't generate a report.

You can specify the reporter option with the `--reporter` flag or the `ci.reporter` config option.

```bash
# Run the CI with an expanded JSON report
unlighthouse-ci --site <your-site> --reporter jsonExpanded
```

```ts unlighthouse.config.ts
export default {
ci: {
reporter: 'jsonExpanded'
}
}
```

### Budget assertions

Unlighthouse simplifies budget assertions. You can provide a single budget number which will be used
Expand Down Expand Up @@ -77,27 +99,28 @@ Configuring the CLI can be done either through the CI arguments or through a con

### CI Options

| Options | |
|------------------------|-----------------------------------------------------------------------------------------|
| `-v, --version` | Display version number. |
| `--site <url>` | Host URL to scan. |
| `--root <path>` | Define the project root. |
| `--config-file <path>` | Path to config file. |
| `--output-path <path>` | Path to save the contents of the client and reports to. |
| `--budget <number>` | Budget (1-100), the minimum score which can pass. |
| `--build-static` | Build a static website for the reports which can be uploaded. |
| `--cache` | Enable the caching. |
| `--no-cache` | Disable the caching. |
| `--throttle` | Enable the throttling. |
| `--samples` | Specify the amount of samples to run. |
| `--urls` | Specify explicit relative URLs as a comma-seperated list. |
| `--router-prefix` | The URL path prefix for the client and API to run from. |
| `--enable-javascript` | When inspecting the HTML wait for the javascript to execute. Useful for SPAs. |
| `--disable-javascript` | When inspecting the HTML, don't wait for the javascript to execute. |
| `--enable-i18n-pages` | Enable scanning pages which use x-default. |
| `--disable-i18n-pages` | Disable scanning pages which use x-default. |
| `-d, --debug` | Debug. Enable debugging in the logger. |
| `-h, --help` | Display available CLI options |
| Options | |
|-------------------------|-------------------------------------------------------------------------------|
| `-v, --version` | Display version number. |
| `--site <url>` | Host URL to scan. |
| `--root <path>` | Define the project root. |
| `--config-file <path>` | Path to config file. |
| `--output-path <path>` | Path to save the contents of the client and reports to. |
| `--budget <number>` | Budget (1-100), the minimum score which can pass. |
| `--reporter <reporter>` | Which reporter to use. Options are jsonSimple and jsonExpanded. |
| `--build-static` | Build a static website for the reports which can be uploaded. |
| `--cache` | Enable the caching. |
| `--no-cache` | Disable the caching. |
| `--throttle` | Enable the throttling. |
| `--samples` | Specify the amount of samples to run. |
| `--urls` | Specify explicit relative URLs as a comma-seperated list. |
| `--router-prefix` | The URL path prefix for the client and API to run from. |
| `--enable-javascript` | When inspecting the HTML wait for the javascript to execute. Useful for SPAs. |
| `--disable-javascript` | When inspecting the HTML, don't wait for the javascript to execute. |
| `--enable-i18n-pages` | Enable scanning pages which use x-default. |
| `--disable-i18n-pages` | Disable scanning pages which use x-default. |
| `-d, --debug` | Debug. Enable debugging in the logger. |
| `-h, --help` | Display available CLI options |


### Config File
Expand Down
108 changes: 51 additions & 57 deletions packages/cli/src/ci.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { join } from 'node:path'
import type { UserConfig } from '@unlighthouse/core'
import fs from 'fs-extra'
import { createUnlighthouse, generateClient, useLogger, useUnlighthouse } from '@unlighthouse/core'
Expand All @@ -8,6 +7,7 @@ import { handleError } from './errors'
import type { CiOptions } from './types'
import { pickOptions, validateHost, validateOptions } from './util'
import createCli from './createCli'
import { generateReportPayload, outputReport } from './reporters'

async function run() {
const startTime = new Date()
Expand All @@ -16,6 +16,7 @@ async function run() {

cli.option('--budget <budget>', 'Budget (1-100), the minimum score which can pass.')
cli.option('--build-static <build-static>', 'Build a static website for the reports which can be uploaded.')
cli.option('--reporter <reporter>', 'The report to generate from results. Options: jsonSimple, jsonExpanded or false. Default is jsonSimple.')

const { options } = cli.parse() as unknown as { options: CiOptions }

Expand All @@ -26,6 +27,7 @@ async function run() {
resolvedOptions.ci = {
budget: options.budget || undefined,
buildStatic: options.buildStatic || false,
reporter: options.reporter || 'jsonSimple',
}

await createUnlighthouse({
Expand Down Expand Up @@ -64,67 +66,59 @@ async function run() {
let hadError = false
if (hasBudget) {
logger.info('Running score budgets.', resolvedConfig.ci.budget)
worker
.reports()
.forEach((report) => {
const categories = report.report?.categories
if (!categories)
return

Object.values(categories).forEach((category) => {
let budget = resolvedConfig.ci.budget
if (!Number.isInteger(budget)) {
// @ts-expect-error need to fix
budget = resolvedConfig.ci.budget[category.key]
}
if (category.score && (category.score * 100) < budget) {
logger.error(`${report.route.path} has invalid score \`${category.score}\` for category \`${category.key}\`.`)
hadError = true
}
})
worker.reports().forEach((report) => {
const categories = report.report?.categories
if (!categories)
return

Object.values(categories).forEach((category: { score: number; key: string }) => {
let budget = resolvedConfig.ci.budget
if (!Number.isInteger(budget))
budget = resolvedConfig.ci.budget[category.key]

if (category.score && category.score * 100 < (budget as number)) {
logger.error(
`${report.route.path} has invalid score \`${category.score}\` for category \`${category.key}\`.`,
)
hadError = true
}
})
})
if (!hadError)
logger.success('Score assertions have passed.')
}
if (resolvedConfig.ci.reporter) {
const reporter = resolvedConfig.ci.reporter
// @ts-expect-error untyped
const payload = generateReportPayload(reporter, worker.reports())
const path = relative(resolvedConfig.root, await outputReport(reporter, resolvedConfig, payload))
logger.success(`Generated \`${resolvedConfig.ci.reporter}\` report \`./${path}\``)
}
if (!hadError) {
logger.success('CI assertions on score budget has passed.')

await fs.writeJson(join(resolvedConfig.root, resolvedConfig.outputPath, 'ci-result.json'),
worker.reports()
.map((report) => {
return {
path: report.route.path,
score: report.report?.score,
}
})
// make the list ordering consistent
.sort((a, b) => a.path.localeCompare(b.path)),
)

if (resolvedConfig.ci?.buildStatic) {
logger.info('Generating static report.')
const { runtimeSettings, resolvedConfig } = useUnlighthouse()
await generateClient({ static: true })
// delete the json lighthouse payloads, we don't need them for the static mode
const globby = (await import('globby'))
const jsonPayloads = await globby.globby(['lighthouse.json', '**/lighthouse.json', 'assets/lighthouse.fbx'], { cwd: runtimeSettings.generatedClientPath, absolute: true })
logger.debug(`Deleting ${jsonPayloads.length} files not required for static build.`)
for (const k in jsonPayloads)
await fs.rm(jsonPayloads[k])

const relativeDir = `./${relative(resolvedConfig.root, runtimeSettings.generatedClientPath)}`
logger.success(`Static report is ready for uploading: \`${relativeDir}\``)
if (!isCI) {
// tell the user they can preview it using sirv-cli and link them to the docs
logger.info(`You can preview the static report using \`npx sirv-cli ${relativeDir}\`.`)
logger.info('For deployment demos, see https://unlighthouse.com/docs/deployment')
}
if (resolvedConfig.ci?.buildStatic) {
logger.info('Generating static report.')
const { runtimeSettings, resolvedConfig } = useUnlighthouse()
await generateClient({ static: true })
// delete the json lighthouse payloads, we don't need them for the static mode
const globby = await import('globby')
const jsonPayloads = await globby.globby(
['lighthouse.json', '**/lighthouse.json', 'assets/lighthouse.fbx'],
{ cwd: runtimeSettings.generatedClientPath, absolute: true },
)
logger.debug(
`Deleting ${jsonPayloads.length} files not required for static build.`,
)
for (const k in jsonPayloads) await fs.rm(jsonPayloads[k])

const relativeDir = `./${relative(resolvedConfig.root, runtimeSettings.generatedClientPath)}`
logger.success(`Static report is ready for uploading: \`${relativeDir}\``)
if (!isCI) {
// tell the user they can preview it using sirv-cli and link them to the docs
logger.info(`You can preview the static report using \`npx sirv-cli ${relativeDir}\`.`)
logger.info('For deployment demos, see https://unlighthouse.com/docs/deployment')
}

process.exit(0)
}
else {
logger.error('Some routes failed the budget.')
process.exit(1)
}
process.exit(0)
})
}

Expand Down
24 changes: 24 additions & 0 deletions packages/cli/src/reporters/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import { join } from 'node:path'
import fse from 'fs-extra'
import type { ResolvedUserConfig, UnlighthouseRouteReport } from '@unlighthouse/core'
import { reportJsonSimple } from './jsonSimple'
import { reportJsonExpanded } from './jsonExpanded'
import type { ReportJsonExpanded, ReportJsonSimple } from './types'

export function generateReportPayload(reporter: 'jsonExpanded', reports: UnlighthouseRouteReport[]): ReportJsonExpanded
export function generateReportPayload(reporter: 'jsonSimple', reports: UnlighthouseRouteReport[]): ReportJsonSimple
export function generateReportPayload(reporter: string, reports: UnlighthouseRouteReport[]): any {
if (reporter.startsWith('json'))
return reporter === 'jsonSimple' ? reportJsonSimple(reports) : reportJsonExpanded(reports)

throw new Error(`Unsupported reporter: ${reporter}.`)
}

export async function outputReport(reporter: string, config: Partial<ResolvedUserConfig>, payload: any) {
if (reporter.startsWith('json')) {
const path = join(config.root, config.outputPath, 'ci-result.json')
await fse.writeJson(path, payload, { spaces: 2 })
return path
}
throw new Error(`Unsupported reporter: ${reporter}.`)
}
141 changes: 141 additions & 0 deletions packages/cli/src/reporters/jsonExpanded.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import type { CategoryAverageScore, CategoryScore, ExpandedRouteReport, MetricAverageScore, MetricScore, ReportJsonExpanded } from './types'

const relevantMetrics = [
'largest-contentful-paint',
'cumulative-layout-shift',
'first-contentful-paint',
'total-blocking-time',
'max-potential-fid',
'interactive',
]

export function reportJsonExpanded(unlighthouseRouteReports): ReportJsonExpanded {
const routes = unlighthouseRouteReports
.map((report) => {
const categories = Object.values(report.report?.categories ?? {}).reduce(
(prev: { [key: string]: CategoryScore }, category: any): any => ({
...prev,
[category.key]: {
key: category.key,
id: category.id,
title: category.title,
score: category.score,
},
}),
{},
)
const metrics = Object.values(report.report?.audits ?? {})
.filter((metric: any) => relevantMetrics.includes(metric.id))
.reduce((prev: { [key: string]: any }, metric: any): any => ({
...prev,
[metric.id]: {
id: metric.id,
title: metric.title,
description: metric.description,
numericValue: metric.numericValue,
numericUnit: metric.numericUnit,
displayValue: metric.displayValue,
},
}), {})
return <ExpandedRouteReport> {
path: report.route.path,
score: report.report?.score,
categories,
metrics,
}
})
// make the list ordering consistent
.sort((a, b) => a.path.localeCompare(b.path))

const averageCategories = extractCategoriesFromRoutes(routes)
const averageMetrics = extractMetricsFromRoutes(routes)

const summary = {
score: parseFloat(
(
routes.reduce((prev, curr) => prev + curr.score, 0) / routes.length
).toFixed(2),
),
categories: averageCategories,
metrics: averageMetrics,
}
return {
summary,
routes,
}
}

function extractCategoriesFromRoutes(routes: ExpandedRouteReport[]) {
const categoriesWithAllScores = routes.reduce((prev, curr) => {
return Object.keys(curr.categories).reduce((target, categoryKey) => {
const scores = target[categoryKey] ? target[categoryKey].scores : []
return {
...target,
[categoryKey]: {
...curr.categories[categoryKey],
scores: [...scores, curr.categories[categoryKey].score],
},
}
}, prev)
}, {} as { [key: string]: { key: string; id: string; title: string; scores: number[] } })

// returns averageCategories
return Object.keys(categoriesWithAllScores).reduce(
(
prev: {
[key: string]: CategoryAverageScore
},
key: string,
) => {
const averageScore = parseFloat(
(
categoriesWithAllScores[key].scores.reduce(
(prev, curr) => prev + curr,
0,
) / categoriesWithAllScores[key].scores.length
).toFixed(2),
)
const { ...strippedCategory }
= categoriesWithAllScores[key]
return { ...prev, [key]: { ...strippedCategory, averageScore } }
},
{} as {
[key: string]: CategoryAverageScore
},
)
}

function extractMetricsFromRoutes(routes: ExpandedRouteReport[]) {
const metricsWithAllNumericValues = routes.reduce((prev, curr) => {
return Object.keys(curr.metrics).reduce((target, metricKey) => {
const numericValues = target[metricKey]
? target[metricKey].numericValues
: []
return {
...target,
[metricKey]: {
...curr.metrics[metricKey],
numericValues: [...numericValues, curr.metrics[metricKey].numericValue],
},
}
}, prev)
}, {} as { [key: string]: Omit<MetricScore, 'numericValue'> & { numericValues: number[] } })

// average metrics
return Object.keys(metricsWithAllNumericValues).reduce(
(prev: { [key: string]: MetricAverageScore }, key: string) => {
const averageNumericValue = parseFloat(
(
metricsWithAllNumericValues[key].numericValues.reduce(
(prev, curr) => prev + curr,
0,
) / metricsWithAllNumericValues[key].numericValues.length
).toFixed(2),
)
const { ...strippedMetric }
= metricsWithAllNumericValues[key]
return { ...prev, [key]: { ...strippedMetric, averageNumericValue } }
},
{} as { [key: string]: MetricAverageScore },
)
}
Loading

0 comments on commit 1cf833e

Please sign in to comment.