diff --git a/docs/cli.md b/docs/cli.md index baf5557e4d3..17cd02040a2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -790,7 +790,7 @@ poetry source add --priority=explicit pypi * `--default`: Set this source as the [default]({{< relref "repositories#default-package-source" >}}) (disable PyPI). Deprecated in favor of `--priority`. * `--secondary`: Set this source as a [secondary]({{< relref "repositories#secondary-package-sources" >}}) source. Deprecated in favor of `--priority`. -* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information. +* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), [`supplemental`]({{< relref "repositories#supplemental-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information. {{% note %}} At most one of the options above can be provided. See [package sources]({{< relref "repositories#package-sources" >}}) for more information. diff --git a/docs/dependency-specification.md b/docs/dependency-specification.md index 1d0d0b51a53..991d6e78510 100644 --- a/docs/dependency-specification.md +++ b/docs/dependency-specification.md @@ -258,7 +258,7 @@ you can use the `source` property: [[tool.poetry.source]] name = "foo" url = "https://foo.bar/simple/" -priority = "secondary" +priority = "supplemental" [tool.poetry.dependencies] my-cool-package = { version = "*", source = "foo" } diff --git a/docs/repositories.md b/docs/repositories.md index d79d88ac7c1..59830f6e7f2 100644 --- a/docs/repositories.md +++ b/docs/repositories.md @@ -29,11 +29,11 @@ By default, Poetry discovers and installs packages from [PyPI](https://pypi.org) install a dependency to your project for a [simple API repository](#simple-api-repository)? Let's do it. -First, [configure](#project-configuration) the [package source](#package-source) as a [secondary package source](#secondary-package-sources) to your +First, [configure](#project-configuration) the [package source](#package-source) as a [supplemental](#supplemental-package-sources) (or [explicit](#explicit-package-sources)) package source to your project. ```bash -poetry source add --priority=secondary foo https://pypi.example.org/simple/ +poetry source add --priority=supplemental foo https://pypi.example.org/simple/ ``` Then, assuming the repository requires authentication, configure credentials for it. @@ -123,13 +123,14 @@ url = "https://foo.bar/simple/" priority = "primary" ``` -If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary and explicit sources. +If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary, supplemental and explicit sources. Package sources are considered in the following order: 1. [default source](#default-package-source), 2. primary sources, 3. implicit PyPI (unless disabled by another [default source](#default-package-source) or configured explicitly), -4. [secondary sources](#secondary-package-sources), +4. [secondary sources](#secondary-package-sources) (DEPRECATED), +5. [supplemental sources](#supplemental-package-sources). [Explicit sources](#explicit-package-sources) are considered only for packages that explicitly [indicate their source](#package-source-constraint). @@ -182,10 +183,12 @@ as a package source for your project. {{% /warning %}} -#### Secondary Package Sources +#### Secondary Package Sources (DEPRECATED) + +*Deprecated in 1.5.0* If package sources are configured as secondary, all it means is that these will be given a lower -priority when selecting compatible package distribution that also exists in your default and primary package sources. +priority when selecting compatible package distribution that also exists in your default and primary package sources. If the package source should instead be searched only if higher-priority repositories did not return results, please consider a [supplemental source](#supplemental-package-sources) instead. You can configure a package source as a secondary source with `priority = "secondary"` in your package source configuration. @@ -196,6 +199,28 @@ poetry source add --priority=secondary https://foo.bar/simple/ There can be more than one secondary package source. +{{% warning %}} + +Secondary package sources are deprecated in favor of supplemental package sources. + +{{% /warning %}} + +#### Supplemental Package Sources + +*Introduced in 1.5.0* + +Package sources configured as supplemental are only searched if no other (higher-priority) source yields a compatible package distribution. This is particularly convenient if the response time of the source is high and relatively few package distributions are to be fetched from this source. + +You can configure a package source as a supplemental source with `priority = "supplemental"` in your package +source configuration. + +```bash +poetry source add --priority=supplemental https://foo.bar/simple/ +``` + +There can be more than one supplemental package source. + + #### Explicit Package Sources *Introduced in 1.5.0* @@ -212,7 +237,7 @@ There can be more than one explicit package source. #### Package Source Constraint -All package sources (including secondary sources) will be searched during the package lookup +All package sources (including secondary and possibly supplemental sources) will be searched during the package lookup process. These network requests will occur for all sources, regardless of if the package is found at one or more sources. diff --git a/src/poetry/console/commands/source/add.py b/src/poetry/console/commands/source/add.py index 3752f340990..1997deb5328 100644 --- a/src/poetry/console/commands/source/add.py +++ b/src/poetry/console/commands/source/add.py @@ -114,6 +114,14 @@ def handle(self) -> int: else: priority = Priority[priority_str.upper()] + if priority is Priority.SECONDARY: + allowed_prios = (p for p in Priority if p is not Priority.SECONDARY) + self.line_error( + "Warning: Priority 'secondary' is deprecated. Consider" + " changing the priority to one of the non-deprecated values:" + f" {', '.join(repr(p.name.lower()) for p in allowed_prios)}." + ) + sources = AoT([]) new_source = Source(name=name, url=url, priority=priority) is_new_source = True diff --git a/src/poetry/factory.py b/src/poetry/factory.py index 9c2c92c93b2..c79135d1077 100644 --- a/src/poetry/factory.py +++ b/src/poetry/factory.py @@ -155,6 +155,16 @@ def create_pool( elif source.get("secondary"): priority = Priority.SECONDARY + if priority is Priority.SECONDARY: + allowed_prios = (p for p in Priority if p is not Priority.SECONDARY) + warning = ( + "Found deprecated priority 'secondary' for source" + f" '{source.get('name')}' in pyproject.toml. Consider changing the" + " priority to one of the non-deprecated values:" + f" {', '.join(repr(p.name.lower()) for p in allowed_prios)}." + ) + io.write_error_line(f"Warning: {warning}") + if io.is_debug(): message = f"Adding repository {repository.name} ({repository.url})" if priority is Priority.DEFAULT: diff --git a/src/poetry/json/schemas/poetry.json b/src/poetry/json/schemas/poetry.json index 930d73bffb4..93a822d28b9 100644 --- a/src/poetry/json/schemas/poetry.json +++ b/src/poetry/json/schemas/poetry.json @@ -45,6 +45,7 @@ "primary", "default", "secondary", + "supplemental", "explicit" ], "description": "Declare the priority of this repository." diff --git a/src/poetry/repositories/repository_pool.py b/src/poetry/repositories/repository_pool.py index 9b9c5e03260..cd839521a35 100644 --- a/src/poetry/repositories/repository_pool.py +++ b/src/poetry/repositories/repository_pool.py @@ -28,6 +28,7 @@ class Priority(IntEnum): DEFAULT = enum.auto() PRIMARY = enum.auto() SECONDARY = enum.auto() + SUPPLEMENTAL = enum.auto() EXPLICIT = enum.auto() @@ -186,11 +187,13 @@ def find_packages(self, dependency: Dependency) -> list[Package]: packages: list[Package] = [] for repo in self.repositories: + if packages and self.get_priority(repo.name) is Priority.SUPPLEMENTAL: + break packages += repo.find_packages(dependency) return packages def search(self, query: str) -> list[Package]: results: list[Package] = [] - for repository in self.repositories: - results += repository.search(query) + for repo in self.repositories: + results += repo.search(query) return results diff --git a/tests/console/commands/source/conftest.py b/tests/console/commands/source/conftest.py index bc86ff400c5..7e44e280069 100644 --- a/tests/console/commands/source/conftest.py +++ b/tests/console/commands/source/conftest.py @@ -51,6 +51,15 @@ def source_secondary() -> Source: ) +@pytest.fixture +def source_supplemental() -> Source: + return Source( + name="supplemental", + url="https://supplemental.com", + priority=Priority.SUPPLEMENTAL, + ) + + @pytest.fixture def source_explicit() -> Source: return Source( @@ -151,6 +160,7 @@ def add_all_source_types( source_primary: Source, source_default: Source, source_secondary: Source, + source_supplemental: Source, source_explicit: Source, ) -> None: add = command_tester_factory("source add", poetry=poetry_with_source) @@ -158,6 +168,7 @@ def add_all_source_types( source_primary, source_default, source_secondary, + source_supplemental, source_explicit, ]: add.execute(f"{source.name} {source.url} --priority={source.name}") diff --git a/tests/console/commands/source/test_add.py b/tests/console/commands/source/test_add.py index 2c3cda9afdd..01c24a37e1e 100644 --- a/tests/console/commands/source/test_add.py +++ b/tests/console/commands/source/test_add.py @@ -28,11 +28,20 @@ def assert_source_added_legacy( source_existing: Source, source_added: Source, ) -> None: + secondary_deprecated_str = ( + "" + if source_added.priority is not Priority.SECONDARY + else ( + "\nWarning: Priority 'secondary' is deprecated. Consider changing the" + " priority to one of the non-deprecated values: 'default', 'primary'," + " 'supplemental', 'explicit'." + ) + ) assert ( tester.io.fetch_error().strip() - == "Warning: Priority was set through a deprecated flag" - " (--default or --secondary). Consider using --priority next" - " time." + == "Warning: Priority was set through a deprecated flag (--default or" + " --secondary). Consider using --priority next time." + + secondary_deprecated_str ) assert ( tester.io.fetch_output().strip() @@ -136,6 +145,20 @@ def test_source_add_secondary( assert_source_added(tester, poetry_with_source, source_existing, source_secondary) +def test_source_add_supplemental( + tester: CommandTester, + source_existing: Source, + source_supplemental: Source, + poetry_with_source: Poetry, +) -> None: + tester.execute( + f"--priority=supplemental {source_supplemental.name} {source_supplemental.url}" + ) + assert_source_added( + tester, poetry_with_source, source_existing, source_supplemental + ) + + def test_source_add_explicit( tester: CommandTester, source_existing: Source, diff --git a/tests/console/commands/source/test_show.py b/tests/console/commands/source/test_show.py index 2aac023fba1..91993a62557 100644 --- a/tests/console/commands/source/test_show.py +++ b/tests/console/commands/source/test_show.py @@ -121,6 +121,7 @@ def test_source_show_two( "source_primary", "source_default", "source_secondary", + "source_supplemental", "source_explicit", ), ) diff --git a/tests/fixtures/with_supplemental_source/pyproject.toml b/tests/fixtures/with_supplemental_source/pyproject.toml new file mode 100644 index 00000000000..00ae71beb1a --- /dev/null +++ b/tests/fixtures/with_supplemental_source/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "with-explicit-source" +version = "1.2.3" +description = "Some description." +authors = [ + "Your Name " +] +license = "MIT" + +# Requirements +[tool.poetry.dependencies] +python = "^3.6" + +[tool.poetry.dev-dependencies] + +[[tool.poetry.source]] +name = "supplemental" +url = "https://supplemental.com/simple/" +priority = "supplemental" diff --git a/tests/json/test_schema_sources.py b/tests/json/test_schema_sources.py index 22a735e6c60..b1f023be2fd 100644 --- a/tests/json/test_schema_sources.py +++ b/tests/json/test_schema_sources.py @@ -30,7 +30,7 @@ def test_pyproject_toml_invalid_priority() -> None: assert Factory.validate(content) == { "errors": [ "[source.0.priority] 'arbitrary' is not one of ['primary', 'default'," - " 'secondary', 'explicit']" + " 'secondary', 'supplemental', 'explicit']" ], "warnings": [], } diff --git a/tests/repositories/test_repository_pool.py b/tests/repositories/test_repository_pool.py index 01b33d9da72..9ec5f68401e 100644 --- a/tests/repositories/test_repository_pool.py +++ b/tests/repositories/test_repository_pool.py @@ -81,9 +81,10 @@ def test_repository_from_single_repo_pool_legacy( assert pool.get_priority("foo") == expected_priority -def test_repository_with_normal_default_secondary_and_explicit_repositories() -> None: +def test_repository_with_all_prio_repositories() -> None: secondary = LegacyRepository("secondary", "https://secondary.com") default = LegacyRepository("default", "https://default.com") + supplemental = LegacyRepository("supplemental", "https://supplemental.com") repo1 = LegacyRepository("foo", "https://foo.bar") repo2 = LegacyRepository("bar", "https://bar.baz") explicit = LegacyRepository("explicit", "https://bar.baz") @@ -92,6 +93,7 @@ def test_repository_with_normal_default_secondary_and_explicit_repositories() -> pool.add_repository(repo1) pool.add_repository(secondary, priority=Priority.SECONDARY) pool.add_repository(repo2) + pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL) pool.add_repository(explicit, priority=Priority.EXPLICIT) pool.add_repository(default, priority=Priority.DEFAULT) @@ -99,11 +101,25 @@ def test_repository_with_normal_default_secondary_and_explicit_repositories() -> assert pool.repository("default") is default assert pool.repository("foo") is repo1 assert pool.repository("bar") is repo2 + assert pool.repository("supplemental") is supplemental assert pool.repository("explicit") is explicit assert pool.has_default() assert pool.has_primary_repositories() +def test_repository_secondary_and_supplemental_repositories_do_show() -> None: + secondary = LegacyRepository("secondary", "https://secondary.com") + supplemental = LegacyRepository("supplemental", "https://supplemental.com") + + pool = RepositoryPool() + pool.add_repository(secondary, priority=Priority.SECONDARY) + pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL) + + assert pool.repository("secondary") is secondary + assert pool.repository("supplemental") is supplemental + assert pool.repositories == [secondary, supplemental] + + def test_repository_explicit_repositories_do_not_show() -> None: explicit = LegacyRepository("explicit", "https://explicit.com") default = LegacyRepository("default", "https://default.com") @@ -174,9 +190,11 @@ def test_repository_ordering() -> None: secondary1 = LegacyRepository("secondary1", "https://secondary1.com") secondary2 = LegacyRepository("secondary2", "https://secondary2.com") secondary3 = LegacyRepository("secondary3", "https://secondary3.com") + supplemental = LegacyRepository("supplemental", "https://supplemental.com") pool = RepositoryPool() pool.add_repository(secondary1, priority=Priority.SECONDARY) + pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL) pool.add_repository(primary1) pool.add_repository(default1, priority=Priority.DEFAULT) pool.add_repository(primary2) @@ -188,7 +206,14 @@ def test_repository_ordering() -> None: pool.add_repository(primary3) pool.add_repository(secondary3, priority=Priority.SECONDARY) - assert pool.repositories == [default1, primary1, primary3, secondary1, secondary3] + assert pool.repositories == [ + default1, + primary1, + primary3, + secondary1, + secondary3, + supplemental, + ] with pytest.raises(ValueError): pool.add_repository(default2, priority=Priority.DEFAULT) @@ -207,11 +232,32 @@ def test_pool_get_package_in_any_repository() -> None: assert returned_package2 == package2 +def test_pool_find_packages_only_considers_supplemental_when_needed() -> None: + package1 = get_package("foo", "1.1.1") + package2 = get_package("foo", "1.2.3") + package3 = get_package("foo", "2.0.0") + repo1 = Repository("repo1", [package1, package3]) + repo2 = Repository("repo2", [package1, package2]) + pool = RepositoryPool([repo1]).add_repository(repo2, priority=Priority.SUPPLEMENTAL) + + dependency_in_nonsupplemental = get_dependency("foo", "^1.0.0") + returned_packages_in_nonsupplemental = pool.find_packages( + dependency_in_nonsupplemental + ) + dependency_needs_supplemental = get_dependency("foo", "1.2.3") + returned_packages_needs_supplemental = pool.find_packages( + dependency_needs_supplemental + ) + + assert returned_packages_in_nonsupplemental == [package1] + assert returned_packages_needs_supplemental == [package2] + + def test_pool_get_package_in_specified_repository() -> None: package = get_package("foo", "1.0.0") - repo1 = Repository("repo1") + repo1 = Repository("repo1", [package]) repo2 = Repository("repo2", [package]) - pool = RepositoryPool([repo1, repo2]) + pool = RepositoryPool([repo1]).add_repository(repo2, priority=Priority.SUPPLEMENTAL) returned_package = pool.package( "foo", Version.parse("1.0.0"), repository_name="repo2" diff --git a/tests/test_factory.py b/tests/test_factory.py index 0667c134227..b57030709ea 100644 --- a/tests/test_factory.py +++ b/tests/test_factory.py @@ -298,7 +298,7 @@ def test_poetry_with_non_default_secondary_source_legacy( assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT assert poetry.pool.has_repository("foo") assert isinstance(poetry.pool.repository("foo"), LegacyRepository) - assert [repo.name for repo in poetry.pool.repositories] == ["PyPI", "foo"] + assert {repo.name for repo in poetry.pool.repositories} == {"PyPI", "foo"} def test_poetry_with_non_default_secondary_source( @@ -311,7 +311,7 @@ def test_poetry_with_non_default_secondary_source( assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT assert poetry.pool.has_repository("foo") assert isinstance(poetry.pool.repository("foo"), LegacyRepository) - assert [repo.name for repo in poetry.pool.repositories] == ["PyPI", "foo"] + assert {repo.name for repo in poetry.pool.repositories} == {"PyPI", "foo"} def test_poetry_with_non_default_multiple_secondary_sources_legacy( @@ -400,7 +400,12 @@ def test_poetry_with_non_default_multiple_sources_pypi( expected = ["bar", "PyPI", "baz", "foo"] assert [repo.name for repo in poetry.pool.repositories] == expected error = io.fetch_error() - assert error == "" + assert ( + error.strip() + == "Warning: Found deprecated priority 'secondary' for source 'foo' in" + " pyproject.toml. Consider changing the priority to one of the" + " non-deprecated values: 'default', 'primary', 'supplemental', 'explicit'." + ) def test_poetry_with_no_default_source(fixture_dir: FixtureDirGetter) -> None: @@ -412,6 +417,20 @@ def test_poetry_with_no_default_source(fixture_dir: FixtureDirGetter) -> None: assert {repo.name for repo in poetry.pool.repositories} == {"PyPI"} +def test_poetry_with_supplemental_source( + fixture_dir: FixtureDirGetter, with_simple_keyring: None +) -> None: + poetry = Factory().create_poetry(fixture_dir("with_supplemental_source")) + + assert poetry.pool.has_repository("PyPI") + assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT + assert isinstance(poetry.pool.repository("PyPI"), PyPiRepository) + assert poetry.pool.has_repository("supplemental") + assert poetry.pool.get_priority("supplemental") is Priority.SUPPLEMENTAL + assert isinstance(poetry.pool.repository("supplemental"), LegacyRepository) + assert {repo.name for repo in poetry.pool.repositories} == {"PyPI", "supplemental"} + + def test_poetry_with_explicit_source( fixture_dir: FixtureDirGetter, with_simple_keyring: None ) -> None: @@ -424,7 +443,7 @@ def test_poetry_with_explicit_source( assert isinstance(poetry.pool.repository("PyPI"), PyPiRepository) assert poetry.pool.has_repository("explicit") assert isinstance(poetry.pool.repository("explicit"), LegacyRepository) - assert [repo.name for repo in poetry.pool.repositories] == ["PyPI"] + assert {repo.name for repo in poetry.pool.repositories} == {"PyPI"} def test_poetry_with_explicit_pypi_and_other(