Support statically linked CPython. (#2472)

This pulls in a new vendored version of Pip with
the recent fix in pypa/pip#12716 applied in
pex-tool/pip#13. As a result, Pex can run using
vendored Pip under statically linked musl libc CPython interpreters.
This opens the door to bootstrapping newer unpatched Pip's that also
have this same fix (versions 24.2 and later; see: #2471).

Fixes #2017

Author: jsirois

pex/vendor/__init__.py

@@ -207,19 +207,24 @@ def iter_vendor_specs(filter_requires_python=None):
     # We shell out to pip at buildtime to resolve and install dependencies.
     # N.B.: We're currently using a patched version of Pip 20.3.4 housed at
     # https://github.com/pex-tool/pip/tree/pex/patches/generation-2.
-    # It has 2 patches:
+    # It has 3 patches:
     # 1.) https://github.com/pex-tool/pip/commit/06f462537c981116c763c1ba40cf40e9dd461bcf
     #     The patch works around a bug in `pip download --constraint...` tracked at
     #     https://github.com/pypa/pip/issues/9283 and fixed by https://github.com/pypa/pip/pull/9301
     #     there and https://github.com/pex-tool/pip/pull/8 in our fork.
     # 2.) https://github.com/pex-tool/pip/commit/386a54f097ece66775d0c7f34fd29bb596c6b0be
     #     This is a cherry-pick of
-    #     https://github.com/pex-tool/pip/commit/00fb5a0b224cde08e3e5ca034247baadfb646468
+    #     https://github.com/pypa/pip/commit/00fb5a0b224cde08e3e5ca034247baadfb646468
     #     (https://github.com/pypa/pip/pull/9533) from upstream that upgrades Pip's vendored
     #     packaging to 20.9 to pick up support for mac universal2 wheels.
+    # 3.) https://github.com/pex-tool/pip/commit/00827ec9f4275a7786425cf006466c56f4cbd862
+    #     This is a cherry-pick of
+    #     https://github.com/pypa/pip/commit/601bcf82eccfbc15c1ff6cc735aafb2c9dab81a5
+    #     (https://github.com/pypa/pip/pull/12716) from upstream that fixes glibc version probing on
+    #     musl libc systems.
     yield VendorSpec.git(
         repo="https://github.com/pex-tool/pip",
-        commit="386a54f097ece66775d0c7f34fd29bb596c6b0be",
+        commit="00827ec9f4275a7786425cf006466c56f4cbd862",
         project_name="pip",
         rewrite=False,
     )

pex/vendor/_vendored/pip/.layout.json

@@ -1 +1 @@
-{"fingerprint": "120267325b80f5c4b4adac019eb6617ab3319395c043d2871eedf70dd6ae2954", "record_relpath": "pip-20.3.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"}
+{"fingerprint": "23b678435eebb1c7423541b5663bdc518833a4b75a3a3a2192466a37cd7b1861", "record_relpath": "pip-20.3.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"}

pex/vendor/_vendored/pip/pip/_internal/utils/glibc.py

@@ -49,7 +49,20 @@ def glibc_version_string_ctypes():
     # manpage says, "If filename is NULL, then the returned handle is for the
     # main program". This way we can let the linker do the work to figure out
     # which libc our process is actually using.
-    process_namespace = ctypes.CDLL(None)
+    #
+    # We must also handle the special case where the executable is not a
+    # dynamically linked executable. This can occur when using musl libc,
+    # for example. In this situation, dlopen() will error, leading to an
+    # OSError. Interestingly, at least in the case of musl, there is no
+    # errno set on the OSError. The single string argument used to construct
+    # OSError comes from libc itself and is therefore not portable to
+    # hard code here. In any case, failure to call dlopen() means we
+    # can't proceed, so we bail on our attempt.
+    try:
+        process_namespace = ctypes.CDLL(None)
+    except OSError:
+        return None
+
     try:
         gnu_get_libc_version = process_namespace.gnu_get_libc_version
     except AttributeError:

tests/integration/test_issue_2017.py

@@ -0,0 +1,76 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import os.path
+import shutil
+import subprocess
+import tarfile
+from textwrap import dedent
+
+import pytest
+
+from pex.common import is_exe, safe_open
+from pex.compatibility import urlparse
+from pex.fetcher import URLFetcher
+from pex.pep_440 import Version
+from pex.pip.version import PipVersion
+from pex.typing import TYPE_CHECKING
+from testing import IS_LINUX, run_pex_command
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+# TODO(John Sirois): Include a test of >= Pip 24.2 when Pex adds support for it.
+#  See: https://github.com/pex-tool/pex/issues/2471
+@pytest.mark.skipif(
+    PipVersion.DEFAULT > PipVersion.VENDORED and PipVersion.DEFAULT.version < Version("24.2"),
+    reason=(
+        "Although Pex's vendored Pip is patched to handle statically linked musl libc CPython, no "
+        "version of Pip Pex supports handles these Pythons until Pip 24.2"
+    ),
+)
+@pytest.mark.skipif(
+    not IS_LINUX,
+    reason="This test tests statically linked musl libc CPython which is only available for Linux.",
+)
+def test_statically_linked_musl_libc_cpython_support(tmpdir):
+    # type: (Any) -> None
+
+    pbs_distribution_url = (
+        "https://github.com/indygreg/python-build-standalone/releases/download/20221220/"
+        "cpython-3.10.9+20221220-x86_64_v3-unknown-linux-musl-install_only.tar.gz"
+    )
+    pbs_distribution = os.path.join(
+        str(tmpdir),
+        os.path.basename(urlparse.urlparse(pbs_distribution_url).path),
+    )
+    with URLFetcher().get_body_stream(pbs_distribution_url) as read_fp, open(
+        pbs_distribution, "wb"
+    ) as write_fp:
+        shutil.copyfileobj(read_fp, write_fp)
+    with tarfile.open(pbs_distribution) as tf:
+        tf.extractall(str(tmpdir))
+    statically_linked_musl_libc_cpython = os.path.join(str(tmpdir), "python", "bin", "python3")
+    assert is_exe(statically_linked_musl_libc_cpython)
+
+    pex = os.path.join(str(tmpdir), "pex")
+    run_pex_command(
+        args=["fortune==1.1.1", "-c", "fortune", "-o", pex],
+        python=statically_linked_musl_libc_cpython,
+    ).assert_success()
+
+    fortune_db = os.path.join(str(tmpdir), "fortunes")
+    with safe_open(fortune_db, "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                A day for firm decisions!!!!!  Or is it?
+                %
+                """
+            )
+        )
+    output = subprocess.check_output(args=[pex, fortune_db])
+    assert b"A day for firm decisions!!!!!  Or is it?\n" == output, output.decode("utf-8")