From dc8a3859cff31a285963410fa05da6148bcd4e94 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sun, 3 Apr 2022 17:55:24 -0400 Subject: [PATCH 1/4] Improve documentation around index restrictions --- docs/advanced.rst | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index fe52751e0f..d710bafdca 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -17,7 +17,10 @@ This document covers some of Pipenv's more glorious and advanced features. ☤ Specifying Package Indexes ---------------------------- -If you'd like a specific package to be installed with a specific package index, you can do the following:: +Starting in release ``2022.3.23`` all packages are mapped only to a single package index for security reasons. +All unspecified packages are resolved using the default index source; the default package index is pypi. + +For a specific package to be installed from an alternate package index, you must match the name of the index as in the following example:: [[source]] url = "https://pypi.org/simple" @@ -25,23 +28,33 @@ If you'd like a specific package to be installed with a specific package index, name = "pypi" [[source]] - url = "http://pypi.home.kennethreitz.org/simple" + url = "https://download.pytorch.org/whl/cu113/" verify_ssl = false - name = "home" + name = "pytorch" [dev-packages] [packages] - requests = {version="*", index="home"} - maya = {version="*", index="pypi"} - records = "*" + torch = {version="*", index="pytorch"} + numpy = {version="*"} + +**Note:** In prior versions of ``pipenv`` it was possible to specify to search ``--extra-index-urls`` to ``pip`` and not +specifically match the expected index by name. This functionality has been deprecated in favor of index restricted +packages,which is a simplifying assumption that is more security mindful. The pip documentation has the following +warning around this option. + +> Using this option to search for packages which are not in the main repository (such as private packages) is unsafe, +per a security vulnerability called dependency confusion: an attacker can claim the package on the public repository +in a way that will ensure it gets chosen over the private package. -Very fancy. +It is possible to use an alternative default index other than pypi. To accomplish this, simply do not specify pypi +as one of the sources in your Pipfile. When pypi is not a default source, any public packages required by your +primary index packages must be mirrored onto your private index or they will not resolve properly, as is standard. ☤ Using a PyPI Mirror ---------------------------- -If you would like to override the default PyPI index URLs with the URL for a PyPI mirror, you can use the following:: +Should you wish to override the default PyPI index URLs with the URL for a PyPI mirror, you can do the following:: $ pipenv install --pypi-mirror @@ -53,9 +66,9 @@ If you would like to override the default PyPI index URLs with the URL for a PyP $ pipenv uninstall --pypi-mirror -Alternatively, you can set the ``PIPENV_PYPI_MIRROR`` environment variable. +Alternatively, setting the ``PIPENV_PYPI_MIRROR`` environment variable is equivalent to passing ``--pypi-mirror ``. -☤ Injecting credentials into Pipfiles via environment variables +☤ Injecting credentials into Pipfile via environment variables ----------------------------------------------------------------- Pipenv will expand environment variables (if defined) in your Pipfile. Quite From ddc30437ef375392e901fa186566d9a28f8f6cc3 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 4 Apr 2022 21:21:38 -0400 Subject: [PATCH 2/4] Update docs/advanced.rst Co-authored-by: Yusuke Nishioka --- docs/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index d710bafdca..602e786bc8 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -40,7 +40,7 @@ For a specific package to be installed from an alternate package index, you must **Note:** In prior versions of ``pipenv`` it was possible to specify to search ``--extra-index-urls`` to ``pip`` and not specifically match the expected index by name. This functionality has been deprecated in favor of index restricted -packages,which is a simplifying assumption that is more security mindful. The pip documentation has the following +packages, which is a simplifying assumption that is more security mindful. The pip documentation has the following warning around this option. > Using this option to search for packages which are not in the main repository (such as private packages) is unsafe, From b3ea7acad5cfeaca11abecb461f88080df1959fb Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 5 Apr 2022 22:53:44 -0400 Subject: [PATCH 3/4] Refine index documentation updates. Factor out and re-use method before closing down other PR. --- docs/advanced.rst | 24 +++++++++++++++++------- news/5022.doc.rst | 1 + news/5022.trivial.rst | 1 + pipenv/utils/resolver.py | 7 ++----- 4 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 news/5022.doc.rst create mode 100644 news/5022.trivial.rst diff --git a/docs/advanced.rst b/docs/advanced.rst index 602e786bc8..09d57334ad 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -18,7 +18,7 @@ This document covers some of Pipenv's more glorious and advanced features. ---------------------------- Starting in release ``2022.3.23`` all packages are mapped only to a single package index for security reasons. -All unspecified packages are resolved using the default index source; the default package index is pypi. +All unspecified packages are resolved using the default index source; the default package index is PyPI. For a specific package to be installed from an alternate package index, you must match the name of the index as in the following example:: @@ -38,18 +38,28 @@ For a specific package to be installed from an alternate package index, you must torch = {version="*", index="pytorch"} numpy = {version="*"} -**Note:** In prior versions of ``pipenv`` it was possible to specify to search ``--extra-index-urls`` to ``pip`` and not -specifically match the expected index by name. This functionality has been deprecated in favor of index restricted +You may install a package such as the example ``torch`` from the named index ``pytorch`` using the CLI by running +the following command: + +``pipenv install --index=pytorch torch`` + +Alternatively the index may be specified by full url, and it will be added to the ``Pipfile`` with a generated name +unless it already exists in which case the existing name with be reused when pinning the package index. + +**Note:** In prior versions of ``pipenv`` you could specify ``--extra-index-urls`` to the ``pip`` resolver and avoid +specifically matching the expected index by name. That functionality was deprecated in favor of index restricted packages, which is a simplifying assumption that is more security mindful. The pip documentation has the following -warning around this option. +warning around the ``--extra-index-urls`` option:: > Using this option to search for packages which are not in the main repository (such as private packages) is unsafe, per a security vulnerability called dependency confusion: an attacker can claim the package on the public repository in a way that will ensure it gets chosen over the private package. -It is possible to use an alternative default index other than pypi. To accomplish this, simply do not specify pypi -as one of the sources in your Pipfile. When pypi is not a default source, any public packages required by your -primary index packages must be mirrored onto your private index or they will not resolve properly, as is standard. +Should you wish to use an alternative default index other than PyPI: simply do not specify PyPI as one of the +sources in your ``Pipfile``. When PyPI is omitted, then any public packages required either directly or +as sub-dependencies must be mirrored onto your private index or they will not resolve properly. This matches the +standard recommendation of ``pip`` maintainers: "To correctly make a private project installable is to point +--index-url to an index that contains both PyPI and their private projects—which is our recommended best practice." ☤ Using a PyPI Mirror ---------------------------- diff --git a/news/5022.doc.rst b/news/5022.doc.rst new file mode 100644 index 0000000000..5c3ab51e88 --- /dev/null +++ b/news/5022.doc.rst @@ -0,0 +1 @@ +Improve documentation around extra indexes and index restricted packages. diff --git a/news/5022.trivial.rst b/news/5022.trivial.rst new file mode 100644 index 0000000000..3dafada899 --- /dev/null +++ b/news/5022.trivial.rst @@ -0,0 +1 @@ +Reuse existing utility method for determining if index is pypi, reducing code complexity. diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 093fb82cd5..e883f65ace 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -26,7 +26,7 @@ translate_markers, ) from .indexes import parse_indexes, prepare_pip_source_args -from .internet import _get_requests_session +from .internet import _get_requests_session, is_pypi_url from .locking import format_requirement_for_lockfile, prepare_lockfile from .shell import make_posix, subprocess_run, temp_environ from .spinner import create_spinner @@ -744,10 +744,7 @@ def collect_hashes(self, ireq): sources = list( filter(lambda s: s.get("name") == self.index_lookup[ireq.name], sources) ) - if any( - "python.org" in source["url"] or "pypi.org" in source["url"] - for source in sources - ): + if any(is_pypi_url(source["url"]) for source in sources): hashes = self._get_hashes_from_pypi(ireq) if hashes: return hashes From f617f64f17f3dcbb48fdbdc3081e126c0fed85bd Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 5 Apr 2022 23:50:50 -0400 Subject: [PATCH 4/4] Fully remove the --extra-index-url argument --- news/5022.removal.rst | 2 ++ pipenv/cli/command.py | 1 - pipenv/cli/options.py | 20 +------------------ pipenv/core.py | 10 +++------- .../notpip/_internal/cli/req_command.py | 3 --- .../patched/notpip/_internal/req/req_file.py | 2 -- 6 files changed, 6 insertions(+), 32 deletions(-) create mode 100644 news/5022.removal.rst diff --git a/news/5022.removal.rst b/news/5022.removal.rst new file mode 100644 index 0000000000..ba89801a8c --- /dev/null +++ b/news/5022.removal.rst @@ -0,0 +1,2 @@ +Removes the optional ``install`` argument ``--extra-index-url`` as it was not compatible with index restricted packages. +Using the ``--index`` argument is the correct way to specify a package should be pulled from the non-default index. diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 99dd30a159..0fa2be37d7 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -236,7 +236,6 @@ def install(state, **kwargs): keep_outdated=state.installstate.keep_outdated, selective_upgrade=state.installstate.selective_upgrade, index_url=state.index, - extra_index_url=state.extra_index_urls, packages=state.installstate.packages, editable_packages=state.installstate.editables, site_packages=state.site_packages, diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 2e1e92778a..110f37d847 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -59,7 +59,6 @@ def main(self, *args, **kwargs): class State: def __init__(self): self.index = None - self.extra_index_urls = [] self.verbose = False self.quiet = False self.pypi_mirror = None @@ -111,28 +110,12 @@ def callback(ctx, param, value): "--index", expose_value=False, envvar="PIP_INDEX_URL", - help="Target PyPI-compatible package index url.", + help="Specify target package index by url or index name from Pipfile.", nargs=1, callback=callback, )(f) -def extra_index_option(f): - def callback(ctx, param, value): - state = ctx.ensure_object(State) - state.extra_index_urls.extend(list(value)) - return value - - return option( - "--extra-index-url", - multiple=True, - expose_value=False, - help="URLs to the extra PyPI compatible indexes to query for package look-ups.", - callback=callback, - envvar="PIP_EXTRA_INDEX_URL", - )(f) - - def editable_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) @@ -630,7 +613,6 @@ def sync_options(f): def install_options(f): f = sync_options(f) f = index_option(f) - f = extra_index_option(f) f = requirementstxt_option(f) f = selective_upgrade_option(f) f = ignore_pipfile_option(f) diff --git a/pipenv/core.py b/pipenv/core.py index 53d247470b..d30ff5bb03 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1923,7 +1923,6 @@ def do_install( packages=False, editable_packages=False, index_url=False, - extra_index_url=False, dev=False, three=False, python=False, @@ -2165,7 +2164,6 @@ def do_install( pre=pre, requirements_dir=requirements_directory, index=index_url, - extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) if c.returncode: @@ -2240,13 +2238,11 @@ def do_install( ) ) # Add the package to the Pipfile. - indexes = list(filter(None, [index_url, *extra_index_url])) - for index in indexes: + if index_url: index_name = project.add_index_to_pipfile( - index, verify_ssl=index.startswith("https:") + index_url, verify_ssl=index_url.startswith("https:") ) - if index_url and not extra_index_url: - pkg_requirement.index = index_name + pkg_requirement.index = index_name try: project.add_package_to_pipfile(pkg_requirement, dev) except ValueError: diff --git a/pipenv/patched/notpip/_internal/cli/req_command.py b/pipenv/patched/notpip/_internal/cli/req_command.py index 17c2209ff6..11582fb187 100644 --- a/pipenv/patched/notpip/_internal/cli/req_command.py +++ b/pipenv/patched/notpip/_internal/cli/req_command.py @@ -62,9 +62,6 @@ def _get_index_urls(cls, options: Values) -> Optional[List[str]]: url = getattr(options, "index_url", None) if url: index_urls.append(url) - urls = getattr(options, "extra_index_urls", None) - if urls: - index_urls.extend(urls) # Return None rather than an empty list return index_urls or None diff --git a/pipenv/patched/notpip/_internal/req/req_file.py b/pipenv/patched/notpip/_internal/req/req_file.py index 2ea94f64c9..804e7006ef 100644 --- a/pipenv/patched/notpip/_internal/req/req_file.py +++ b/pipenv/patched/notpip/_internal/req/req_file.py @@ -225,8 +225,6 @@ def handle_option_line( index_urls = [opts.index_url] if opts.no_index is True: index_urls = [] - if opts.extra_index_urls: - index_urls.extend(opts.extra_index_urls) if opts.find_links: # FIXME: it would be nice to keep track of the source # of the find_links: support a find-links local path