feat(pwa): add request filtering for recorded sections (#682)

* feat(pwa): allow filtering recorded requests

* fix(pwa): deprecate old config options

* docs(pwa): warn about breaking changes to experimental features

cli/config/d2.pwa.config.js

@@ -12,15 +12,24 @@ module.exports = {
  enabled: false,
  caching: {
  /**
- * If true, don't cache requests to exteral domains by default.
+ * If true, don't cache requests to exteral domains in app shell.
  * Doesn't affect recording mode
  */
+ omitExternalRequestsFromAppShell: false,
+ /** Deprecated version of above */
  omitExternalRequests: false,
  /**
- * Don't cache URLs matching patterns in this array by default.
+ * Don't cache URLs matching patterns in this array in app shell.
  * Doesn't affect recording mode
  */
+ patternsToOmitFromAppShell: [],
+ /** Deprecated version of above */
  patternsToOmit: [],
+ /**
+ * Don't cache URLs matching these patterns in recorded sections.
+ * Can still be cached in app shell unless filtered there too.
+ */
+ patternsToOmitFromCacheableSections: [],
  /**
  * In addition to the contents of an app's 'build' folder, other
  * URLs can be precached by adding them to this list which will

cli/src/lib/pwa/getPWAEnvVars.js

@@ -1,11 +1,18 @@
+/** Preps string literals for regex conversion by escaping special chars */
+function escapeForRegex(string) {
+ return string.replace(/[-\\^$*+?.()|[\]{}]/g, '\\$&')
+}
+
 /**
- * Handles patterns defined as strings or RegExps
+ * Handles patterns defined as strings or RegExps and stringifies the array
+ * for passing as an env var. Note that all patterns will be converted to
+ * Regexes in the service worker.
  * @param {Object} config
  */
-function getPatternsToOmit(config) {
- const patternsToOmit = config.pwa.caching.patternsToOmit.map(pattern => {
+function stringifyPatterns(patternsList) {
+ const stringsList = patternsList.map(pattern => {
  if (typeof pattern === 'string') {
- return pattern
+ return escapeForRegex(pattern)
  } else if (pattern instanceof RegExp) {
  return pattern.source
  } else {
@@ -14,7 +21,7 @@ function getPatternsToOmit(config) {
  )
  }
  })
- return JSON.stringify(patternsToOmit)
+ return JSON.stringify(stringsList)
 }
 
 /**
@@ -29,10 +36,23 @@ function getPWAEnvVars(config) {
  }
  return {
  pwa_enabled: JSON.stringify(config.pwa.enabled),
+ pwa_caching_omit_external_requests_from_app_shell: JSON.stringify(
+ config.pwa.caching.omitExternalRequestsFromAppShell
+ ),
+ // Deprecated version of the above:
  pwa_caching_omit_external_requests: JSON.stringify(
  config.pwa.caching.omitExternalRequests
  ),
- pwa_caching_patterns_to_omit: getPatternsToOmit(config),
+ pwa_caching_patterns_to_omit_from_app_shell: stringifyPatterns(
+ config.pwa.caching.patternsToOmitFromAppShell
+ ),
+ // Deprecated version of the above:
+ pwa_caching_patterns_to_omit: stringifyPatterns(
+ config.pwa.caching.patternsToOmit
+ ),
+ pwa_caching_patterns_to_omit_from_cacheable_sections: stringifyPatterns(
+ config.pwa.caching.patternsToOmitFromCacheableSections
+ ),
  }
 }
 

docs/pwa/pwa.md

@@ -1,20 +1,23 @@
 # Progressive Web App tools
 
-!> **WARNING** These features are for advanced purposes only and have some significant drawbacks in their first implementation. Keep an eye out for a more robust implementation in the future!
-
 The App Platform provides some tools that can enable some PWA features and offline caching.
 
+!> **WARNING** These features are considered **experimental** and are **subject to breaking changes outside of the normal release cycle.** They are for advanced purposes only and have some significant drawbacks in their first implementation. Keep an eye out for a more robust implementation in the future!
+
 ### Opting In
 
 You can opt in to PWA features using options in `d2.config.js`. Here are the options and their effects:
 
-| **`config` Property** | **Type** | **Description** |
-| --------------------------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `pwa.enabled` | Boolean | If **true**, enables registration of a service worker to perform offline caching in both development and production builds. In development mode, the service worker uses different caching strategies to facilitate development; [see below](#in-development). If **false**, any service worker registered in this scope will be **unregistered**. |
-| `pwa.caching` | Object | Contains several properties to configure offline caching by the service worker; see the definitions of the following properties below. |
-| `pwa.caching.omitExternalRequests` | Boolean | If **true**, omits requests to external domains from the default caching strategies. If **false** (default), requests to external domains will be cached by the default strategies. Note that _all requests are cached during recording mode_, regardless of this setting. |
-| `pwa.caching.patternsToOmit` | Array of RegExps or Strings | A list of URL patterns to omit from the default caching strategies. Strings will be converted to RegExes using `new RegExp(str)` to test URLs. If a URL matches one of these patterns, that request will not be cached by the default caching strategies. Note that _all requests are cached during recording mode_, regardless of this setting. When choosing these URL filters, note that it is better to _cache too many things_ than to risk _not caching an important part of the app shell_, so choose your filter patterns accordingly. |
-| `pwa.caching.additionalManifestEntries` | Array of Objects with signature `{ revision: String, url: String }` | A list of files that can be added to the precache manifest. Note that the service worker uses Workbox to precache all static assets that end up in the ‘build’ folder after the CRA compilation and build step during the d2-app-scripts build process. The format of this list must match the [required format for Workbox precache manifests](https://developers.google.com/web/tools/workbox/modules/workbox-precaching#explanation_of_the_precache_list), i.e. it must include a revision hash to inform when that file needs to be updated in the precache. |
+| **`config` Property** | **Type** | **Description** |
+| ------------------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `pwa.enabled` | Boolean | If **true**, enables registration of a service worker to perform offline caching in both development and production builds. In development mode, the service worker uses different caching strategies to facilitate development; [see below](#in-development). If **false**, any service worker registered in this scope will be **unregistered**. |
+| `pwa.caching` | Object | Contains several properties to configure offline caching by the service worker; see the definitions of the following properties below. |
+| `pwa.caching.omitExternalRequestsFromAppShell` | Boolean | If **true**, omits requests to external domains from the default app shell caching strategies. If **false** (default), requests to external domains will be cached in the app shell. Note that this setting does not affect the recording mode. |
+| `pwa.caching.omitExternalRequests` | Boolean | Deprecated; superceded by `omitExternalRequestsFromAppShell`. The new option takes precedence. |
+| `pwa.caching.patternsToOmitFromAppShell` | Array of RegExps or Strings | A list of URL patterns to omit from the default app shell caching strategies. Strings will be converted to RegExes using `new RegExp(str)` (with their special characters escaped) to test URLs. If a URL matches one of these patterns, that request will not be cached as part of the app shell. Note that this setting does not affect the recording mode. When choosing these URL filters, note that it is better to _cache too many things_ than to risk _not caching an important part of the app shell_ which could break the offline functionality of the app, so choose your filter patterns accordingly. |
+| `pwa.caching.patternsToOmitFromCacheableSections` | Array of RegExps or Strings | Similar to the above setting, except this is a list of URL patterns to omit from _cacheable (recorded) sections_. Requests with URLs that are filtered out from cacheable sections can still be cached in the app shell cache, unless they are filtered out from the app shell as well using the setting above. When choosing these URL filters, note that it is better to _cache too many things_ than to risk _not caching an important part of the section_ which could break the offline functionality of the section, so choose your filter patterns accordingly. |
+| `pwa.caching.patternsToOmit` | Array of RegExps or Strings | Deprecated; superceded by `patternsToOmitFromAppShell`. The new option takes precedence. |
+| `pwa.caching.additionalManifestEntries` | Array of Objects with signature `{ revision: String, url: String }` | A list of files that can be added to the precache manifest. Note that the service worker uses Workbox to precache all static assets that end up in the ‘build’ folder after the CRA compilation and build step during the d2-app-scripts build process. The format of this list must match the [required format for Workbox precache manifests](https://developers.google.com/web/tools/workbox/modules/workbox-precaching#explanation_of_the_precache_list), i.e. it must include a revision hash to inform when that file needs to be updated in the precache. |
 
 ### Offline caching
 

examples/pwa-app/d2.config.js

@@ -5,7 +5,7 @@ const config = {
  enabled: true,
  caching: {
  // For the purposes of this demo, to simulate dashboard content:
- patternsToOmit: ['visualizations'],
+ patternsToOmitFromAppShell: ['visualizations'],
  },
  },
 

pwa/src/service-worker/recording-mode.js

@@ -1,6 +1,13 @@
 import { swMsgs } from '../lib/constants'
 import { openSectionsDB, SECTIONS_STORE } from '../lib/sections-db'
 
+// '[]' Fallback prevents error when switching from pwa enabled to disabled
+const CACHEABLE_SECTION_URL_FILTER_PATTERNS = JSON.parse(
+ process.env
+ .REACT_APP_DHIS2_APP_PWA_CACHING_PATTERNS_TO_OMIT_FROM_CACHEABLE_SECTIONS ||
+ '[]'
+).map(pattern => new RegExp(pattern))
+
 // Triggered on 'START_RECORDING' message
 export function startRecording(event) {
  console.debug('[SW] Starting recording')
@@ -40,13 +47,33 @@ function isClientRecording(clientId) {
  return clientId in self.clientRecordingStates
 }
 
-/** Used to check if requests should be handled by recording handler */
-export function isClientRecordingRequests(clientId) {
+/**
+ * A request-matching function for the recorded request route. If 'true' is
+ * returned, the request will be handled by the handler below.
+ */
+export function shouldRequestBeRecorded({ url, event }) {
+ const clientId = event.clientId
+ // If not recording, don't handle
+ if (!isClientRecording(clientId)) {
+ return false
+ }
+
  // Don't record requests when waiting for completion confirmation
- return (
- isClientRecording(clientId) &&
- self.clientRecordingStates[clientId].confirmationTimeout === undefined
+ if (
+ self.clientRecordingStates[clientId].confirmationTimeout !== undefined
+ ) {
+ return false
+ }
+
+ // Don't cache if url matches filter in pattern list from d2.config.js
+ const urlMatchesFilter = CACHEABLE_SECTION_URL_FILTER_PATTERNS.some(
+ pattern => pattern.test(url.href)
  )
+ if (urlMatchesFilter) {
+ return false
+ }
+
+ return true
 }
 
 /** Request handler during recording mode */

pwa/src/service-worker/service-worker.js

@@ -10,10 +10,10 @@ import {
  startRecording,
  completeRecording,
  handleRecordedRequest,
- isClientRecordingRequests,
+ shouldRequestBeRecorded,
 } from './recording-mode'
 import {
- urlMeetsDefaultCachingCriteria,
+ urlMeetsAppShellCachingCriteria,
  createDB,
  removeUnusedCaches,
  setUpKillSwitchServiceWorker,
@@ -133,25 +133,23 @@ export function setUpServiceWorker() {
 
  // Request handler during recording mode: ALL requests are cached
  // Handling routing: https://developers.google.com/web/tools/workbox/modules/workbox-routing#matching_and_handling_in_routes
- registerRoute(
- ({ event }) => isClientRecordingRequests(event.clientId),
- handleRecordedRequest
- )
+ registerRoute(shouldRequestBeRecorded, handleRecordedRequest)
 
- // If not recording, fall through to default caching strategies
+ // If not recording, fall through to default caching strategies for app
+ // shell:
  // SWR strategy for static assets that can't be precached.
- // Skip in development environments
+ // (Skip in development environments)
  registerRoute(
  ({ url }) =>
  PRODUCTION_ENV &&
- urlMeetsDefaultCachingCriteria(url) &&
+ urlMeetsAppShellCachingCriteria(url) &&
  fileExtensionRegexp.test(url.pathname),
  new StaleWhileRevalidate({ cacheName: 'other-assets' })
  )
 
  // Network-first caching by default
  registerRoute(
- ({ url }) => urlMeetsDefaultCachingCriteria(url),
+ ({ url }) => urlMeetsAppShellCachingCriteria(url),
  new NetworkFirst({ cacheName: 'app-shell' })
  )