fix: robots.txt Allow rule breaking filtering

harlan-zw · Jan 23, 2024 · fe7a6ab · fe7a6ab
1 parent e4c8115
commit fe7a6ab
Showing 1 changed file with 8 additions and 5 deletions.
diff --git a/packages/core/src/discovery/robotsTxt.ts b/packages/core/src/discovery/robotsTxt.ts
@@ -72,11 +72,14 @@ export function mergeRobotsTxtConfig(config: ResolvedUserConfig, { groups, sitem
  ...(config.scanner.exclude || []),
  ...normalisedGroups.flatMap(group => group.disallow),
  ])].filter(isValidRegex)
- config.scanner.include = [...new Set([
- ...(config.scanner.include || []),
- ...normalisedGroups.flatMap(group => group.allow),
- ])].filter(isValidRegex)
-
+ config.scanner.include = config.scanner.include || []
+ const robotsAllows = normalisedGroups.flatMap(group => group.allow).filter(a => a.length)
+ if (!config.scanner.include.length && robotsAllows.length) {
+ config.scanner.include = [...new Set([
+ '/*',
+ ...normalisedGroups.flatMap(group => group.allow),
+ ])].filter(isValidRegex)
+ }
  if (config.scanner.sitemap !== false && sitemaps.length)
  config.scanner.sitemap = [...new Set([...(Array.isArray(config.scanner.sitemap) ? config.scanner.sitemap : []), ...sitemaps])]
 }