feat: add obsolete snapshot reporting (#222)

Co-authored-by: Andres Escobar <[email protected]>

README.md

@@ -160,6 +160,28 @@ expect.extend({ toMatchImageSnapshot });
 ### jest.retryTimes()
 Jest supports [automatic retries on test failures](https://jestjs.io/docs/en/jest-object#jestretrytimes). This can be useful for browser screenshot tests which tend to have more frequent false positives. Note that when using jest.retryTimes you'll have to use a unique customSnapshotIdentifier as that's the only way to reliably identify snapshots.
 
+### Removing Outdated Snapshots
+
+Unlike jest-managed snapshots, the images created by `jest-image-snapshot` will not be automatically removed by the `-u` flag if they are no longer needed. You can force `jest-image-snapshot` to remove the files by including the `outdated-snapshot-reporter` in your config and running with the environment variable `JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE`.
+
+```json
+{
+ "jest": {
+ "reporters": [
+ "default",
+ "jest-image-snapshot/src/outdated-snapshot-reporter.js"
+ ]
+ }
+}
+```
+
+**WARNING: Do not run a *partial* test suite with this flag as it may consider snapshots of tests that weren't run to be obsolete.**
+
+```bash
+export JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE=1
+jest
+```
+
 ### Recommendations when using SSIM comparison
 Since SSIM calculates differences in structural similarity by building a moving 'window' over an images pixels, it does not particularly benefit from pixel count comparisons, especially when you factor in that it has a lot of floating point arithmetic in javascript. However, SSIM gains two key benefits over pixel by pixel comparison:
 - Reduced false positives (failing tests when the images look the same)

__tests__/outdated-snapshot-reporter.spec.js

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2020 American Express Travel Related Services Company, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+const fs = require('fs');
+const os = require('os');
+const childProcess = require('child_process');
+const path = require('path');
+const rimraf = require('rimraf');
+
+describe('OutdatedSnapshotReporter', () => {
+ const jestImageSnapshotDir = path.join(__dirname, '..');
+ const imagePath = path.join(__dirname, 'stubs/TestImage.png');
+ const jestBinPath = path.join(jestImageSnapshotDir, 'node_modules/.bin/jest');
+ let tmpDir = os.tmpdir();
+
+ function setupTestProject(dir) {
+ const jestConfig = {
+ reporters: [
+ 'default',
+ `${jestImageSnapshotDir}/src/outdated-snapshot-reporter.js`,
+ ],
+ };
+ const jestConfigFile = `module.exports = ${JSON.stringify(jestConfig)}`;
+
+ const commonTest = `
+ const fs = require('fs');
+ const {toMatchImageSnapshot} = require('${jestImageSnapshotDir}');
+ expect.extend({toMatchImageSnapshot});
+ `;
+ const imageTest = `${commonTest}
+ it('should run an image snapshot test', () => {
+ expect(fs.readFileSync('image.png')).toMatchImageSnapshot();
+ });
+ `;
+ const doubleTest = `${imageTest}
+ it('should run an image snapshot test', () => {
+ expect(fs.readFileSync('image.png')).toMatchImageSnapshot();
+ });
+ `;
+
+ fs.writeFileSync(path.join(dir, 'jest.config.js'), jestConfigFile);
+ fs.writeFileSync(path.join(dir, 'image.test.js'), imageTest);
+ fs.writeFileSync(path.join(dir, 'double.test.js'), doubleTest);
+ fs.copyFileSync(imagePath, path.join(dir, 'image.png'));
+ }
+
+ function runJest(cliArgs, environment = {}) {
+ return childProcess.spawnSync(jestBinPath, cliArgs, {
+ cwd: tmpDir,
+ encoding: 'utf-8',
+ env: { ...process.env, ...environment },
+ });
+ }
+
+ function getSnapshotFiles() {
+ return fs.readdirSync(path.join(tmpDir, '__image_snapshots__'));
+ }
+
+ beforeAll(() => {
+ tmpDir = fs.mkdtempSync(
+ path.join(os.tmpdir(), 'jest-image-snapshot-tests')
+ );
+ setupTestProject(tmpDir);
+ });
+
+ afterAll(() => {
+ rimraf.sync(tmpDir);
+ });
+
+ it('should write the image snapshot on first run', () => {
+ const { status, stdout, stderr } = runJest(['-u']);
+ expect(stderr).toContain('snapshots written');
+ expect(status).toEqual(0);
+ expect(stdout).toEqual('');
+
+ expect(getSnapshotFiles()).toHaveLength(3);
+ });
+
+ it('should not delete the snapshot when environment flag is not enabled', () => {
+ const { status, stdout } = runJest(['-u', 'image.test.js']);
+ expect(status).toEqual(0);
+ expect(stdout).toEqual('');
+
+ expect(getSnapshotFiles()).toHaveLength(3);
+ });
+
+ it('should delete the snapshot when environment flag is enabled', () => {
+ const { status, stdout, stderr } = runJest(['-u', 'image.test.js'], {
+ JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE: '1',
+ });
+ expect(stderr).toContain('outdated snapshot');
+ expect(status).toEqual(0);
+ expect(stdout).toEqual('');
+
+ expect(getSnapshotFiles()).toHaveLength(1);
+ });
+});

src/index.js

@@ -18,6 +18,7 @@ const path = require('path');
 const Chalk = require('chalk').constructor;
 const { diffImageToSnapshot, runDiffImageToSnapshot } = require('./diff-snapshot');
 const fs = require('fs');
+const OutdatedSnapshotReporter = require('./outdated-snapshot-reporter');
 
 const timesCalled = new Map();
 
@@ -180,6 +181,7 @@ function configureToMatchImageSnapshot({
  const snapshotsDir = customSnapshotsDir || path.join(path.dirname(testPath), SNAPSHOTS_DIR);
  const diffDir = customDiffDir || path.join(snapshotsDir, '__diff_output__');
  const baselineSnapshotPath = path.join(snapshotsDir, `${snapshotIdentifier}-snap.png`);
+ OutdatedSnapshotReporter.markTouchedFile(baselineSnapshotPath);
 
  if (snapshotState._updateSnapshot === 'none' && !fs.existsSync(baselineSnapshotPath)) {
  return {

src/outdated-snapshot-reporter.js

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2020 American Express Travel Related Services Company, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/* eslint-disable class-methods-use-this */
+
+const fs = require('fs');
+const path = require('path');
+
+const TOUCHED_FILE_LIST_PATH = path.join(
+ process.cwd(),
+ '.jest-image-snapshot-touched-files'
+);
+
+const IS_ENABLED = !!process.env.JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE;
+
+class OutdatedSnapshotReporter {
+ /* istanbul ignore next - test coverage in child process */
+ static markTouchedFile(filePath) {
+ if (!IS_ENABLED) return;
+ const touchedListFileDescriptor = fs.openSync(TOUCHED_FILE_LIST_PATH, 'as');
+ fs.writeSync(touchedListFileDescriptor, `${filePath}\n`);
+ fs.closeSync(touchedListFileDescriptor);
+ }
+
+ /* istanbul ignore next - test coverage in child process */
+ static readTouchedFileListFromDisk() {
+ if (!fs.existsSync(TOUCHED_FILE_LIST_PATH)) return [];
+
+ return Array.from(
+ new Set(
+ fs
+ .readFileSync(TOUCHED_FILE_LIST_PATH, 'utf-8')
+ .split('\n')
+ .filter(file => file && fs.existsSync(file))
+ )
+ );
+ }
+
+ /* istanbul ignore next - test coverage in child process */
+ onRunStart() {
+ if (!IS_ENABLED) return;
+ if (fs.existsSync(TOUCHED_FILE_LIST_PATH)) {
+ fs.unlinkSync(TOUCHED_FILE_LIST_PATH);
+ }
+ }
+
+ /* istanbul ignore next - test coverage in child process */
+ onRunComplete() {
+ if (!IS_ENABLED) return;
+ const touchedFiles = OutdatedSnapshotReporter.readTouchedFileListFromDisk();
+ const imageSnapshotDirectories = Array.from(
+ new Set(touchedFiles.map(file => path.dirname(file)))
+ );
+ const allFiles = imageSnapshotDirectories
+ .map(dir => fs.readdirSync(dir).map(file => path.join(dir, file)))
+ .reduce((a, b) => a.concat(b), [])
+ .filter(file => file.endsWith('-snap.png'));
+ const obsoleteFiles = allFiles.filter(
+ file => !touchedFiles.includes(file)
+ );
+
+ if (fs.existsSync(TOUCHED_FILE_LIST_PATH)) {
+ fs.unlinkSync(TOUCHED_FILE_LIST_PATH);
+ }
+
+ obsoleteFiles.forEach((file) => {
+ process.stderr.write(`Deleting outdated snapshot "${file}"...\n`);
+ fs.unlinkSync(file);
+ });
+ }
+}
+
+module.exports = OutdatedSnapshotReporter;