6713 Introduce supplemental package source

docs/cli.md

@@ -790,7 +790,7 @@ poetry source add --priority=explicit pypi
 
 * `--default`: Set this source as the [default]({{< relref "repositories#default-package-source" >}}) (disable PyPI). Deprecated in favor of `--priority`.
 * `--secondary`: Set this source as a [secondary]({{< relref "repositories#secondary-package-sources" >}}) source. Deprecated in favor of `--priority`.
-* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information.
+* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), [`supplemental`]({{< relref "repositories#supplemental-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information.
 
 {{% note %}}
 At most one of the options above can be provided. See [package sources]({{< relref "repositories#package-sources" >}}) for more information.

docs/dependency-specification.md

@@ -258,7 +258,7 @@ you can use the `source` property:
 [[tool.poetry.source]]
 name = "foo"
 url = "https://foo.bar/simple/"
-priority = "secondary"
+priority = "supplemental"
 
 [tool.poetry.dependencies]
 my-cool-package = { version = "*", source = "foo" }

docs/repositories.md

@@ -29,11 +29,11 @@ By default, Poetry discovers and installs packages from [PyPI](https://pypi.org)
 install a dependency to your project for a [simple API repository](#simple-api-repository)? Let's
 do it.
 
-First, [configure](#project-configuration) the [package source](#package-source) as a [secondary package source](#secondary-package-sources) to your
+First, [configure](#project-configuration) the [package source](#package-source) as a [supplemental](#supplemental-package-sources) (or [explicit](#explicit-package-sources)) package source to your
 project.
 
 ```bash
-poetry source add --priority=secondary foo https://pypi.example.org/simple/
+poetry source add --priority=supplemental foo https://pypi.example.org/simple/
 ```
 
 Then, assuming the repository requires authentication, configure credentials for it.
@@ -123,13 +123,14 @@ url = "https://foo.bar/simple/"
 priority = "primary"
 ```
 
-If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary and explicit sources.
+If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary, supplemental and explicit sources.
 
 Package sources are considered in the following order:
 1. [default source](#default-package-source),
 2. primary sources,
 3. implicit PyPI (unless disabled by another [default source](#default-package-source) or configured explicitly),
-4. [secondary sources](#secondary-package-sources),
+4. [secondary sources](#secondary-package-sources) (DEPRECATED),
+5. [supplemental sources](#supplemental-package-sources).
 
 [Explicit sources](#explicit-package-sources) are considered only for packages that explicitly [indicate their source](#package-source-constraint).
 
@@ -182,10 +183,12 @@ as a package source for your project.
 
 {{% /warning %}}
 
-#### Secondary Package Sources
+#### Secondary Package Sources (DEPRECATED)
+
+*Deprecated in 1.5.0*
 
 If package sources are configured as secondary, all it means is that these will be given a lower
-priority when selecting compatible package distribution that also exists in your default and primary package sources.
+priority when selecting compatible package distribution that also exists in your default and primary package sources. If the package source should instead be searched only if higher-priority repositories did not return results, please consider a [supplemental source](#supplemental-package-sources) instead.
 
 You can configure a package source as a secondary source with `priority = "secondary"` in your package
 source configuration.
@@ -196,6 +199,28 @@ poetry source add --priority=secondary https://foo.bar/simple/
 
 There can be more than one secondary package source.
 
+{{% warning %}}
+
+Secondary package sources are deprecated in favor of supplemental package sources.
+
+{{% /warning %}}
+
+#### Supplemental Package Sources
+
+*Introduced in 1.5.0*
+
+Package sources configured as supplemental are only searched if no other (higher-priority) source yields a compatible package distribution. This is particularly convenient if the response time of the source is high and relatively few package distributions are to be fetched from this source.
+
+You can configure a package source as a supplemental source with `priority = "supplemental"` in your package
+source configuration.
+
+```bash
+poetry source add --priority=supplemental https://foo.bar/simple/
+```
+
+There can be more than one supplemental package source.
+
+
 #### Explicit Package Sources
 
 *Introduced in 1.5.0*
@@ -212,7 +237,7 @@ There can be more than one explicit package source.
 
 #### Package Source Constraint
 
-All package sources (including secondary sources) will be searched during the package lookup
+All package sources (including secondary and possibly supplemental sources) will be searched during the package lookup
 process. These network requests will occur for all sources, regardless of if the package is
 found at one or more sources.
 

src/poetry/console/commands/source/add.py

@@ -114,6 +114,14 @@ def handle(self) -> int:
  else:
  priority = Priority[priority_str.upper()]
 
+ if priority is Priority.SECONDARY:
+ allowed_prios = (p for p in Priority if p is not Priority.SECONDARY)
+ self.line_error(
+ "<warning>Warning: Priority 'secondary' is deprecated. Consider"
+ " changing the priority to one of the non-deprecated values:"
+ f" {', '.join(repr(p.name.lower()) for p in allowed_prios)}.</warning>"
+ )
+
  sources = AoT([])
  new_source = Source(name=name, url=url, priority=priority)
  is_new_source = True

src/poetry/factory.py

@@ -155,6 +155,16 @@ def create_pool(
  elif source.get("secondary"):
  priority = Priority.SECONDARY
 
+ if priority is Priority.SECONDARY:
+ allowed_prios = (p for p in Priority if p is not Priority.SECONDARY)
+ warning = (
+ "Found deprecated priority 'secondary' for source"
+ f" '{source.get('name')}' in pyproject.toml. Consider changing the"
+ " priority to one of the non-deprecated values:"
+ f" {', '.join(repr(p.name.lower()) for p in allowed_prios)}."
+ )
+ io.write_error_line(f"<warning>Warning: {warning}</warning>")
+
  if io.is_debug():
  message = f"Adding repository {repository.name} ({repository.url})"
  if priority is Priority.DEFAULT:

src/poetry/json/schemas/poetry.json

@@ -45,6 +45,7 @@
  "primary",
  "default",
  "secondary",
+ "supplemental",
  "explicit"
  ],
  "description": "Declare the priority of this repository."

src/poetry/repositories/repository_pool.py

@@ -28,6 +28,7 @@ class Priority(IntEnum):
  DEFAULT = enum.auto()
  PRIMARY = enum.auto()
  SECONDARY = enum.auto()
+ SUPPLEMENTAL = enum.auto()
  EXPLICIT = enum.auto()
 
 
@@ -186,11 +187,13 @@ def find_packages(self, dependency: Dependency) -> list[Package]:
 
  packages: list[Package] = []
  for repo in self.repositories:
+ if packages and self.get_priority(repo.name) is Priority.SUPPLEMENTAL:
+ break
  packages += repo.find_packages(dependency)
  return packages
 
  def search(self, query: str) -> list[Package]:
  results: list[Package] = []
- for repository in self.repositories:
- results += repository.search(query)
+ for repo in self.repositories:
+ results += repo.search(query)
  return results

tests/console/commands/source/conftest.py

@@ -51,6 +51,15 @@ def source_secondary() -> Source:
  )
 
 
+@pytest.fixture
+def source_supplemental() -> Source:
+ return Source(
+ name="supplemental",
+ url="https://supplemental.com",
+ priority=Priority.SUPPLEMENTAL,
+ )
+
+
 @pytest.fixture
 def source_explicit() -> Source:
  return Source(
@@ -151,13 +160,15 @@ def add_all_source_types(
  source_primary: Source,
  source_default: Source,
  source_secondary: Source,
+ source_supplemental: Source,
  source_explicit: Source,
 ) -> None:
  add = command_tester_factory("source add", poetry=poetry_with_source)
  for source in [
  source_primary,
  source_default,
  source_secondary,
+ source_supplemental,
  source_explicit,
  ]:
  add.execute(f"{source.name} {source.url} --priority={source.name}")

tests/console/commands/source/test_add.py

@@ -28,11 +28,20 @@ def assert_source_added_legacy(
  source_existing: Source,
  source_added: Source,
 ) -> None:
+ secondary_deprecated_str = (
+ ""
+ if source_added.priority is not Priority.SECONDARY
+ else (
+ "\nWarning: Priority 'secondary' is deprecated. Consider changing the"
+ " priority to one of the non-deprecated values: 'default', 'primary',"
+ " 'supplemental', 'explicit'."
+ )
+ )
  assert (
  tester.io.fetch_error().strip()
- == "Warning: Priority was set through a deprecated flag"
- " (--default or --secondary). Consider using --priority next"
- " time."
+ == "Warning: Priority was set through a deprecated flag (--default or"
+ " --secondary). Consider using --priority next time."
+ + secondary_deprecated_str
  )
  assert (
  tester.io.fetch_output().strip()
@@ -136,6 +145,20 @@ def test_source_add_secondary(
  assert_source_added(tester, poetry_with_source, source_existing, source_secondary)
 
 
+def test_source_add_supplemental(
+ tester: CommandTester,
+ source_existing: Source,
+ source_supplemental: Source,
+ poetry_with_source: Poetry,
+) -> None:
+ tester.execute(
+ f"--priority=supplemental {source_supplemental.name} {source_supplemental.url}"
+ )
+ assert_source_added(
+ tester, poetry_with_source, source_existing, source_supplemental
+ )
+
+
 def test_source_add_explicit(
  tester: CommandTester,
  source_existing: Source,

tests/console/commands/source/test_show.py

@@ -121,6 +121,7 @@ def test_source_show_two(
  "source_primary",
  "source_default",
  "source_secondary",
+ "source_supplemental",
  "source_explicit",
  ),
 )

tests/fixtures/with_supplemental_source/pyproject.toml

@@ -0,0 +1,19 @@
+[tool.poetry]
+name = "with-explicit-source"
+version = "1.2.3"
+description = "Some description."
+authors = [
+ "Your Name <[email protected]>"
+]
+license = "MIT"
+
+# Requirements
+[tool.poetry.dependencies]
+python = "^3.6"
+
+[tool.poetry.dev-dependencies]
+
+[[tool.poetry.source]]
+name = "supplemental"
+url = "https://supplemental.com/simple/"
+priority = "supplemental"

tests/json/test_schema_sources.py

@@ -30,7 +30,7 @@ def test_pyproject_toml_invalid_priority() -> None:
  assert Factory.validate(content) == {
  "errors": [
  "[source.0.priority] 'arbitrary' is not one of ['primary', 'default',"
- " 'secondary', 'explicit']"
+ " 'secondary', 'supplemental', 'explicit']"
  ],
  "warnings": [],
  }

tests/repositories/test_repository_pool.py

@@ -81,9 +81,10 @@ def test_repository_from_single_repo_pool_legacy(
  assert pool.get_priority("foo") == expected_priority
 
 
-def test_repository_with_normal_default_secondary_and_explicit_repositories() -> None:
+def test_repository_with_all_prio_repositories() -> None:
  secondary = LegacyRepository("secondary", "https://secondary.com")
  default = LegacyRepository("default", "https://default.com")
+ supplemental = LegacyRepository("supplemental", "https://supplemental.com")
  repo1 = LegacyRepository("foo", "https://foo.bar")
  repo2 = LegacyRepository("bar", "https://bar.baz")
  explicit = LegacyRepository("explicit", "https://bar.baz")
@@ -92,18 +93,33 @@ def test_repository_with_normal_default_secondary_and_explicit_repositories() ->
  pool.add_repository(repo1)
  pool.add_repository(secondary, priority=Priority.SECONDARY)
  pool.add_repository(repo2)
+ pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL)
  pool.add_repository(explicit, priority=Priority.EXPLICIT)
  pool.add_repository(default, priority=Priority.DEFAULT)
 
  assert pool.repository("secondary") is secondary
  assert pool.repository("default") is default
  assert pool.repository("foo") is repo1
  assert pool.repository("bar") is repo2
+ assert pool.repository("supplemental") is supplemental
  assert pool.repository("explicit") is explicit
  assert pool.has_default()
  assert pool.has_primary_repositories()
 
 
+def test_repository_secondary_and_supplemental_repositories_do_show() -> None:
+ secondary = LegacyRepository("secondary", "https://secondary.com")
+ supplemental = LegacyRepository("supplemental", "https://supplemental.com")
+
+ pool = RepositoryPool()
+ pool.add_repository(secondary, priority=Priority.SECONDARY)
+ pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL)
+
+ assert pool.repository("secondary") is secondary
+ assert pool.repository("supplemental") is supplemental
+ assert pool.repositories == [secondary, supplemental]
+
+
 def test_repository_explicit_repositories_do_not_show() -> None:
  explicit = LegacyRepository("explicit", "https://explicit.com")
  default = LegacyRepository("default", "https://default.com")
@@ -174,9 +190,11 @@ def test_repository_ordering() -> None:
  secondary1 = LegacyRepository("secondary1", "https://secondary1.com")
  secondary2 = LegacyRepository("secondary2", "https://secondary2.com")
  secondary3 = LegacyRepository("secondary3", "https://secondary3.com")
+ supplemental = LegacyRepository("supplemental", "https://supplemental.com")
 
  pool = RepositoryPool()
  pool.add_repository(secondary1, priority=Priority.SECONDARY)
+ pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL)
  pool.add_repository(primary1)
  pool.add_repository(default1, priority=Priority.DEFAULT)
  pool.add_repository(primary2)
@@ -188,7 +206,14 @@ def test_repository_ordering() -> None:
  pool.add_repository(primary3)
  pool.add_repository(secondary3, priority=Priority.SECONDARY)
 
- assert pool.repositories == [default1, primary1, primary3, secondary1, secondary3]
+ assert pool.repositories == [
+ default1,
+ primary1,
+ primary3,
+ secondary1,
+ secondary3,
+ supplemental,
+ ]
  with pytest.raises(ValueError):
  pool.add_repository(default2, priority=Priority.DEFAULT)
 
@@ -207,11 +232,32 @@ def test_pool_get_package_in_any_repository() -> None:
  assert returned_package2 == package2
 
 
+def test_pool_find_packages_only_considers_supplemental_when_needed() -> None:
+ package1 = get_package("foo", "1.1.1")
+ package2 = get_package("foo", "1.2.3")
+ package3 = get_package("foo", "2.0.0")
+ repo1 = Repository("repo1", [package1, package3])
+ repo2 = Repository("repo2", [package1, package2])
+ pool = RepositoryPool([repo1]).add_repository(repo2, priority=Priority.SUPPLEMENTAL)
+
+ dependency_in_nonsupplemental = get_dependency("foo", "^1.0.0")
+ returned_packages_in_nonsupplemental = pool.find_packages(
+ dependency_in_nonsupplemental
+ )
+ dependency_needs_supplemental = get_dependency("foo", "1.2.3")
+ returned_packages_needs_supplemental = pool.find_packages(
+ dependency_needs_supplemental
+ )
+
+ assert returned_packages_in_nonsupplemental == [package1]
+ assert returned_packages_needs_supplemental == [package2]
+
+
 def test_pool_get_package_in_specified_repository() -> None:
  package = get_package("foo", "1.0.0")
- repo1 = Repository("repo1")
+ repo1 = Repository("repo1", [package])
  repo2 = Repository("repo2", [package])
- pool = RepositoryPool([repo1, repo2])
+ pool = RepositoryPool([repo1]).add_repository(repo2, priority=Priority.SUPPLEMENTAL)
 
  returned_package = pool.package(
  "foo", Version.parse("1.0.0"), repository_name="repo2"