feat: Resolve relative URIs in nitpick.styles.include

- Relative URIs are resolved against the URI of the style file they are
  contained in.
- Symlinks in file paths are resolved first.
- During normalisation, `gh:https://` is mapped to `github:https://`, and `py:https://` is mapped to `pypackage:https://` to avoid duplicating
  loading.

docs/nitpick_section.rst

@@ -72,6 +72,28 @@ Example of usage: `Nitpick's default style <https://github.com/andreoliwa/nitpic
  [nitpick.styles]
  include = ["styles/python37", "styles/poetry"]
 
-The styles will be merged following the sequence in the list.
+The styles will be merged following the sequence in the list. The ``.toml``
+extension for each referenced file can be onitted.
+
+Relative references are resolved relative to the URI of the style doument they
+are included in according to the `normal rules of RFC 3986 <https://www.rfc-editor.org/rfc/rfc3986.html#section-5.2>`_.
+
+E.g. for a style file located at
+``gh:https://$GITHUB_TOKEN@foo_dev/bar_project@branchname/styles/foobar.toml`` the following
+strings all reference the exact same canonical location to include:
+
+.. code-block:: toml
+
+ [nitpick.styles]
+ include = [
+ "foobar.toml",
+ "../styles/foobar.toml",
+ "/bar_project@branchname/styles/foobar.toml",
+ "//$GITHUB_TOKEN@foo_dev/bar_project@branchname/styles/foobar.toml",
+ ]
+
+For style files on the local filesystem, the canonical path
+(after symbolic links have been resolved) of the style file is used as the
+base.
 
 If a key/value pair appears in more than one sub-style, it will be overridden; the last declared key/pair will prevail.

src/nitpick/project.py

@@ -43,9 +43,9 @@
 def glob_files(dir_: Path, file_patterns: Iterable[str]) -> set[Path]:
  """Search a directory looking for file patterns."""
  for pattern in file_patterns:
- found_files = list(dir_.glob(pattern))
+ found_files = set(dir_.glob(pattern))
  if found_files:
- return set(found_files)
+ return found_files
  return set()
 
 
@@ -175,7 +175,8 @@ def merge_styles(self, offline: bool) -> Iterator[Fuss]:
  from nitpick.style import StyleManager
 
  style = StyleManager(self, offline, config.cache)
- style_errors = list(style.find_initial_styles(peekable(always_iterable(config.styles))))
+ base = f"file:https://{config.file}" if config.file else None
+ style_errors = list(style.find_initial_styles(peekable(always_iterable(config.styles)), base))
  if style_errors:
  raise QuitComplainingError(style_errors)
 

src/nitpick/style/core.py

@@ -1,12 +1,10 @@
 """Style files."""
 from __future__ import annotations
 
-import os
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
-from typing import Iterator, Set, Type
-from urllib.parse import urljoin, urlsplit, urlunsplit
+from typing import Iterable, Iterator, Sequence, Set, Type
 
 import dpath.util
 from flatten_dict import flatten, unflatten
@@ -21,26 +19,23 @@
 from nitpick.blender import SEPARATOR_FLATTEN, TomlDoc, custom_reducer, custom_splitter, search_json
 from nitpick.constants import (
  CACHE_DIR_NAME,
- DOT_SLASH,
  MERGED_STYLE_TOML,
  NITPICK_STYLE_TOML,
  NITPICK_STYLES_INCLUDE_JMEX,
  PROJECT_NAME,
  PROJECT_OWNER,
  PYPROJECT_TOML,
  SLASH,
- TOML_EXTENSION,
 )
 from nitpick.exceptions import QuitComplainingError, pretty_exception
-from nitpick.generic import is_url
 from nitpick.plugins.base import NitpickPlugin
 from nitpick.plugins.info import FileInfo
 from nitpick.project import Project, glob_files
 from nitpick.schemas import BaseStyleSchema, flatten_marshmallow_errors
 from nitpick.style.config import ConfigValidator
 from nitpick.style.fetchers import Scheme, StyleFetcherManager
 from nitpick.style.fetchers.github import GitHubURL
-from nitpick.typedefs import JsonDict, StrOrIterable, StrOrList, mypy_property
+from nitpick.typedefs import JsonDict, mypy_property
 from nitpick.violations import Fuss, Reporter, StyleViolations
 
 Plugins = Set[Type[NitpickPlugin]]
@@ -58,7 +53,6 @@ def __post_init__(self) -> None:
  """Initialize dependant fields."""
  self._merged_styles: JsonDict = {}
  self._already_included: set[str] = set()
- self._first_full_path: str = ""
  self._dynamic_schema_class: type = BaseStyleSchema
  self._style_fetcher_manager = StyleFetcherManager(self.offline, self.cache_dir, self.cache_option)
  self._config_validator = ConfigValidator(self.project)
@@ -89,38 +83,46 @@ def get_default_style_url(github=False):
  rv = furl(scheme=Scheme.PY, host=PROJECT_NAME, path=SLASH.join(["resources", "presets", PROJECT_NAME]))
  return str(rv)
 
- def find_initial_styles(self, configured_styles: StrOrIterable) -> Iterator[Fuss]:
- """Find the initial style(s) and include them."""
+ def find_initial_styles(self, configured_styles: Sequence[str], base: str | None = None) -> Iterator[Fuss]:
+ """Find the initial style(s) and include them.
+
+ base is the URI for the source of the initial styles, and is used to
+ resolve relative references. If omitted, defaults to the project root.
+
+ """
+ project_root = self.project.root.resolve()
+ base = base or f"file:https://{project_root}/"
+
  if configured_styles:
- chosen_styles: StrOrIterable = list(configured_styles)
+ chosen_styles = configured_styles
  log_message = f"Using styles configured in {PYPROJECT_TOML}"
  else:
- paths = glob_files(self.project.root, [NITPICK_STYLE_TOML])
+ paths = glob_files(project_root, [NITPICK_STYLE_TOML])
  if paths:
- chosen_styles = str(sorted(paths)[0])
+ chosen_styles = [str(sorted(paths)[0])]
  log_message = "Using local style found climbing the directory tree"
  else:
- chosen_styles = self.get_default_style_url()
+ chosen_styles = [self.get_default_style_url()]
  log_message = "Using default remote Nitpick style"
  logger.info(f"{log_message}: {chosen_styles}")
 
- yield from self.include_multiple_styles(chosen_styles)
+ yield from self.include_multiple_styles(
+ self._style_fetcher_manager.normalize_url(uri, base) for uri in chosen_styles
+ )
 
- def include_multiple_styles(self, chosen_styles: StrOrIterable) -> Iterator[Fuss]:
+ def include_multiple_styles(self, chosen_styles: Iterable[str]) -> Iterator[Fuss]:
  """Include a list of styles (or just one) into this style tree."""
- for style_uri in always_iterable(chosen_styles):
+ for style_uri in chosen_styles:
  yield from self._include_style(style_uri)
 
- def _include_style(self, style_uri):
- style_uri = self._normalize_style_uri(style_uri)
- style_path, file_contents = self.get_style_path(style_uri)
- if not style_path:
+ def _include_style(self, style_uri: str) -> Iterator[Fuss]:
+ if style_uri in self._already_included:
  return
+ self._already_included.add(style_uri)
 
- resolved_path = str(style_path.resolve())
- if resolved_path in self._already_included:
+ style_path, file_contents = self.get_style_path(style_uri)
+ if not style_path:
  return
- self._already_included.add(resolved_path)
 
  read_toml_dict = self._read_toml(file_contents, style_path)
 
@@ -129,6 +131,14 @@ def _include_style(self, style_uri):
  except ValueError:
  display_name = style_uri
 
+ # normalize sub-style URIs, before merging
+ sub_styles = [
+ self._style_fetcher_manager.normalize_url(uri, style_uri)
+ for uri in always_iterable(search_json(read_toml_dict, NITPICK_STYLES_INCLUDE_JMEX, []))
+ ]
+ if sub_styles:
+ read_toml_dict.setdefault("nitpick", {}).setdefault("styles", {})["include"] = sub_styles
+
  toml_dict, validation_errors = self._config_validator.validate(read_toml_dict)
 
  if validation_errors:
@@ -138,9 +148,7 @@ def _include_style(self, style_uri):
 
  dpath.util.merge(self._merged_styles, flatten(toml_dict, custom_reducer(SEPARATOR_FLATTEN)))
 
- sub_styles: StrOrList = search_json(toml_dict, NITPICK_STYLES_INCLUDE_JMEX, [])
- if sub_styles:
- yield from self.include_multiple_styles(sub_styles)
+ yield from self.include_multiple_styles(sub_styles)
 
  def _read_toml(self, file_contents, style_path):
  toml = TomlDoc(string=file_contents)
@@ -157,45 +165,8 @@ def _read_toml(self, file_contents, style_path):
  ) from err
  return read_toml_dict
 
- def _normalize_style_uri(self, uri):
- is_current_uri_url = is_url(uri)
- if is_current_uri_url:
- self._first_full_path = uri
- return self._append_toml_extension_url(uri)
-
- uri = self._append_toml_extension(uri)
-
- if not self._first_full_path:
- self._first_full_path = str(Path(uri).resolve().parent) + "/"
- return uri
-
- if self._first_full_path and not uri.startswith(DOT_SLASH):
- if os.path.isabs(uri):
- return uri
-
- return self._join_uri(uri)
-
- return uri
-
- def _join_uri(self, uri):
- if is_url(self._first_full_path):
- return urljoin(self._first_full_path, uri)
-
- return str(Path(self._first_full_path).joinpath(uri))
-
- def _append_toml_extension_url(self, url):
- scheme, netloc, path, query, fragment = urlsplit(url)
- path = self._append_toml_extension(path)
- return urlunsplit((scheme, netloc, path, query, fragment))
-
- @staticmethod
- def _append_toml_extension(path):
- if path.endswith(TOML_EXTENSION):
- return path
- return f"{path}{TOML_EXTENSION}"
-
  def get_style_path(self, style_uri: str) -> tuple[Path | None, str]:
- """Get the style path from the URI. Add the .toml extension if it's missing."""
+ """Get the style path from the URI."""
  clean_style_uri = style_uri.strip()
  return self._style_fetcher_manager.fetch(clean_style_uri)
 

src/nitpick/style/fetchers/__init__.py

@@ -6,8 +6,9 @@
 from functools import lru_cache
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Tuple
-from urllib.parse import urlparse, uses_netloc, uses_relative
+from urllib.parse import uses_netloc, uses_relative
 
+from furl import furl
 from requests_cache import CachedSession
 from strenum import LowercaseStrEnum
 
@@ -16,7 +17,7 @@
 from nitpick.style import parse_cache_option
 
 if TYPE_CHECKING:
- from nitpick.style.fetchers.base import FetchersType
+ from nitpick.style.fetchers.base import FetchersType, StyleFetcher
 
 StyleInfo = Tuple[Optional[Path], str]
 
@@ -54,27 +55,40 @@ def __post_init__(self):
  self.session = CachedSession(self.cache_dir / "styles", expire_after=expire_after, cache_control=cache_control)
  self.fetchers = _get_fetchers(self.session)
 
+ def normalize_url(self, url: str, base: str) -> str:
+ """Normalize a style URL.
+
+ The URL is made absolute against base, then passed to individual fetchers
+ to produce a canonical version of the URL.
+
+ """
+ absolute = furl(base).join(url)
+ return self._fetcher_for(absolute).normalize(absolute)
+
  def fetch(self, url) -> StyleInfo:
- """Determine which fetcher to be used and fetch from it.
+ """Determine which fetcher to be used and fetch from it."""
+ fetcher = self._fetcher_for(url)
+ if self.offline and fetcher.requires_connection:
+ return None, ""
+
+ return fetcher.fetch(url)
+
+ def _fetcher_for(self, url) -> StyleFetcher:
+ """Determine which fetcher to be used.
 
  Try a fetcher by domain first, then by protocol scheme.
+
  """
  domain, scheme = self._get_domain_scheme(url)
- fetcher = None
- if domain:
- fetcher = self.fetchers.get(domain)
+ fetcher = self.fetchers.get(domain) if domain else None
  if not fetcher:
  fetcher = self.fetchers.get(scheme)
  if not fetcher:
  raise RuntimeError(f"URI protocol {scheme!r} is not supported")
-
- if self.offline and fetcher.requires_connection:
- return None, ""
-
- return fetcher.fetch(url)
+ return fetcher
 
  @staticmethod
- def _get_domain_scheme(url: str) -> tuple[str, str]:
+ def _get_domain_scheme(url: str | furl) -> tuple[str, str]:
  r"""Get domain and scheme from an URL or a file.
 
  >>> StyleFetcherManager._get_domain_scheme("/abc")
@@ -88,15 +102,14 @@ def _get_domain_scheme(url: str) -> tuple[str, str]:
  >>> StyleFetcherManager._get_domain_scheme("https://server.com/abc")
  ('server.com', 'http')
  """
- if is_url(url):
- parsed_url = urlparse(url)
-  return parsed_url.hostname or "", parsed_url.scheme
- return "", "file"
+ if isinstance(url, str) and not is_url(url):
+ return "", "file"
+ parsed_url = furl(url)
+ return parsed_url.host or "", parsed_url.scheme or "file"
 
 
 def _get_fetchers(session: CachedSession) -> FetchersType:
  # pylint: disable=import-outside-toplevel
- from nitpick.style.fetchers.base import StyleFetcher
  from nitpick.style.fetchers.file import FileFetcher
  from nitpick.style.fetchers.github import GitHubFetcher
  from nitpick.style.fetchers.http import HttpFetcher

src/nitpick/style/fetchers/base.py

@@ -4,10 +4,13 @@
 from dataclasses import dataclass
 from pathlib import Path
 from typing import ClassVar, Dict
+from urllib.parse import urlparse
 
+from furl import furl
 from requests_cache import CachedSession
 from slugify import slugify
 
+from nitpick.constants import TOML_EXTENSION
 from nitpick.generic import is_url
 from nitpick.style.fetchers import StyleInfo
 
@@ -28,6 +31,26 @@ def __post_init__(self):
  if self.requires_connection and self.session is None:
  raise ValueError("session is required")
 
+ def _normalize_path(self, path: str) -> str: # pylint: disable=no-self-use
+ """Normalize the path component of a URL."""
+ return path if path.endswith(TOML_EXTENSION) else f"{path}{TOML_EXTENSION}"
+
+ def _normalize_scheme(self, scheme: str) -> str: # pylint: disable=no-self-use
+ """Normalize the scheme component of a URL."""
+ return scheme
+
+ def normalize(self, url: furl) -> str:
+ """Normalize a URL.
+
+ Produces a canonical URL, meant to be used to uniquely identify a style resource.
+
+ - The base name has .toml appended if not already ending in that extension
+ - Individual fetchers can further normalize the path and scheme.
+
+ """
+ scheme, path = self._normalize_scheme(url.scheme), self._normalize_path(str(url.path.normalize()))
+ return url.set(scheme=scheme, path=path).url
+
  def fetch(self, url) -> StyleInfo:
  """Fetch a style from a specific fetcher."""
  contents = self._do_fetch(url)
@@ -37,6 +60,9 @@ def fetch(self, url) -> StyleInfo:
 
  @staticmethod
  def _get_output_path(url) -> Path:
+ if url.startswith("file:"):
+ url = urlparse(url).path
+
  if is_url(url):
  return Path(slugify(url))