From 1d44a1dade9d054e31b837a3c8d1b690578bc9aa Mon Sep 17 00:00:00 2001 From: Valentin Haenel Date: Wed, 3 Feb 2016 11:17:29 +0100 Subject: [PATCH 1/6] fix installation using --target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We encountered a problem on CentOS 7 machines when using the `--target` option to install packages that are equipped with c-extensions. Observe: ``` me@box ~ $ uname -a Linux box.d.rz.is 3.10.0-327.4.4.el7.x86_64 #1 SMP Tue Jan 5 16:07:00 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux me@box ~ $ python --version Python 2.7.5 me@box ~ $ virtualenv --version 1.10.1 me@box ~ $ virtualenv venv New python executable in venv/bin/python Installing Setuptools.......................................................................................................................................................done. Installing Pip............................................................................................................................................................................................................................................................................done. me@box ~ $ source venv/bin/activate (venv)me@box ~ $ mkdir target (venv)me@box ~ $ ls target venv (venv)me@box ~ $ pip install --index-url https://my_proxy:5000/dev/dev -U pip Collecting pip Downloading https://my_proxy:5000/root/pypi/+f/205/6f553d5b593d3/pip-8.0.2-py2.py3-none-any.whl (1.2MB) 100% |████████████████████████████████| 1.2MB 14.3MB/s Installing collected packages: pip Found existing installation: pip 1.4.1 Uninstalling pip-1.4.1: Successfully uninstalled pip-1.4.1 Successfully installed pip-8.0.2 (venv)me@box ~ $ pip install --index-url https://my_proxy:5000/dev/dev --target target pyyaml Collecting pyyaml Downloading https://my_proxy:5000/root/pypi/+f/89c/bc92cda979042/PyYAML-3.11.zip (371kB) 100% |████████████████████████████████| 372kB 18.6MB/s Installing collected packages: pyyaml Running setup.py install for pyyaml ... done Successfully installed pyyaml Exception: Traceback (most recent call last): File "/data/home/me/venv/lib/python2.7/site-packages/pip/basecommand.py", line 209, in main status = self.run(options, args) File "/data/home/me/venv/lib/python2.7/site-packages/pip/commands/install.py", line 357, in run for item in os.listdir(lib_dir): OSError: [Errno 2] No such file or directory: '/tmp/tmpz2LoDs/lib/python' (venv)me@box ~ $ pip install --index-url https://my_proxy:5000/dev/dev --target target simplejson Collecting simplejson Downloading https://my_proxy:5000/root/pypi/+f/b84/41f1053edd9dc/simplejson-3.8.1.tar.gz (76kB) 100% |████████████████████████████████| 77kB 19.0MB/s Installing collected packages: simplejson Running setup.py install for simplejson ... done Successfully installed simplejson Exception: Traceback (most recent call last): File "/data/home/me/venv/lib/python2.7/site-packages/pip/basecommand.py", line 209, in main status = self.run(options, args) File "/data/home/me/venv/lib/python2.7/site-packages/pip/commands/install.py", line 357, in run for item in os.listdir(lib_dir): OSError: [Errno 2] No such file or directory: '/tmp/tmpEFNOZq/lib/python' (venv)me@box ~ $ pip install --index-url https://my_proxy:5000/dev/dev --target target markupsafe Collecting markupsafe Downloading https://my_proxy:5000/root/pypi/+f/f5a/b3deee4c37cd6/MarkupSafe-0.23.tar.gz Installing collected packages: markupsafe Running setup.py install for markupsafe ... done Successfully installed markupsafe Exception: Traceback (most recent call last): File "/data/home/me/venv/lib/python2.7/site-packages/pip/basecommand.py", line 209, in main status = self.run(options, args) File "/data/home/me/venv/lib/python2.7/site-packages/pip/commands/install.py", line 357, in run for item in os.listdir(lib_dir): OSError: [Errno 2] No such file or directory: '/tmp/tmp4UuvDJ/lib/python' (venv)me@box ~ $ ls /tmp/tmp4UuvDJ/ lib64 ``` As you can see using `--target` first installs into a temporary directory and then installes into the desired `target`. Well, at least it tries to: for any packages that have c-extensions (in this case pyyaml, simplejson and markupsafe) it incorrectly looks for a `lib` directory whereas it should be looking for a `lib64` directory. This patch fixes that for us by looking in a fallback location. We can confirm that this fixes the issue for us. Please advise. --- pip/commands/install.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pip/commands/install.py b/pip/commands/install.py index 7ddde93c255..c48e2f11bbd 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -353,6 +353,13 @@ def run(self, options, args): ensure_dir(options.target_dir) lib_dir = distutils_scheme('', home=temp_target_dir)['purelib'] + if not os.path.exists(lib_dir): + lib_dir = distutils_scheme('', home=temp_target_dir)['platlib'] + if not os.path.exists(lib_dir): + raise InstallationError( + "Unable to install into target dir: %s.", + options.target_dir + ) for item in os.listdir(lib_dir): target_item_dir = os.path.join(options.target_dir, item) From 00e286fa6c77220f8fbc00266bd097b1f981f9dc Mon Sep 17 00:00:00 2001 From: Valentin Haenel Date: Thu, 11 Feb 2016 11:29:43 +0100 Subject: [PATCH 2/6] install both from purelib and platlib if they exist --- pip/commands/install.py | 75 +++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index c48e2f11bbd..602de055073 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -352,42 +352,43 @@ def run(self, options, args): if options.target_dir: ensure_dir(options.target_dir) - lib_dir = distutils_scheme('', home=temp_target_dir)['purelib'] - if not os.path.exists(lib_dir): - lib_dir = distutils_scheme('', home=temp_target_dir)['platlib'] - if not os.path.exists(lib_dir): - raise InstallationError( - "Unable to install into target dir: %s.", - options.target_dir - ) + lib_dir_purelib = distutils_scheme('', home=temp_target_dir)['purelib'] + lib_dir_platlib = distutils_scheme('', home=temp_target_dir)['platlib'] + if os.path.exists(lib_dir_purelib): + self._install_to_target(lib_dir_purelib) + if os.path.exists(lib_dir_platlib): + self._install_to_target(lib_dir_purelib) - for item in os.listdir(lib_dir): - target_item_dir = os.path.join(options.target_dir, item) - if os.path.exists(target_item_dir): - if not options.upgrade: - logger.warning( - 'Target directory %s already exists. Specify ' - '--upgrade to force replacement.', - target_item_dir - ) - continue - if os.path.islink(target_item_dir): - logger.warning( - 'Target directory %s already exists and is ' - 'a link. Pip will not automatically replace ' - 'links, please remove if replacement is ' - 'desired.', - target_item_dir - ) - continue - if os.path.isdir(target_item_dir): - shutil.rmtree(target_item_dir) - else: - os.remove(target_item_dir) - - shutil.move( - os.path.join(lib_dir, item), - target_item_dir - ) - shutil.rmtree(temp_target_dir) return requirement_set + + @staticmethod + def _install_to_target(lib_dir): + for item in os.listdir(lib_dir): + target_item_dir = os.path.join(options.target_dir, item) + if os.path.exists(target_item_dir): + if not options.upgrade: + logger.warning( + 'Target directory %s already exists. Specify ' + '--upgrade to force replacement.', + target_item_dir + ) + continue + if os.path.islink(target_item_dir): + logger.warning( + 'Target directory %s already exists and is ' + 'a link. Pip will not automatically replace ' + 'links, please remove if replacement is ' + 'desired.', + target_item_dir + ) + continue + if os.path.isdir(target_item_dir): + shutil.rmtree(target_item_dir) + else: + os.remove(target_item_dir) + + shutil.move( + os.path.join(lib_dir, item), + target_item_dir + ) + shutil.rmtree(temp_target_dir) From 34a2d85f2828c68a9455858c0b560f302567cb95 Mon Sep 17 00:00:00 2001 From: Valentin Haenel Date: Thu, 11 Feb 2016 11:39:52 +0100 Subject: [PATCH 3/6] oups.... --- pip/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index 602de055073..21bd7976367 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -357,7 +357,7 @@ def run(self, options, args): if os.path.exists(lib_dir_purelib): self._install_to_target(lib_dir_purelib) if os.path.exists(lib_dir_platlib): - self._install_to_target(lib_dir_purelib) + self._install_to_target(lib_dir_platlib) return requirement_set From 35b73196bc2815d8350f77cffee9df1b153a5e27 Mon Sep 17 00:00:00 2001 From: Valentin Haenel Date: Thu, 11 Feb 2016 14:10:28 +0100 Subject: [PATCH 4/6] fix --- pip/commands/install.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index 21bd7976367..94b0acb5917 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -355,16 +355,16 @@ def run(self, options, args): lib_dir_purelib = distutils_scheme('', home=temp_target_dir)['purelib'] lib_dir_platlib = distutils_scheme('', home=temp_target_dir)['platlib'] if os.path.exists(lib_dir_purelib): - self._install_to_target(lib_dir_purelib) + self._install_to_target(lib_dir_purelib, options.target_dir) if os.path.exists(lib_dir_platlib): - self._install_to_target(lib_dir_platlib) + self._install_to_target(lib_dir_platlib, options.target_dir) return requirement_set @staticmethod - def _install_to_target(lib_dir): + def _install_to_target(lib_dir, target_dir): for item in os.listdir(lib_dir): - target_item_dir = os.path.join(options.target_dir, item) + target_item_dir = os.path.join(target_dir, item) if os.path.exists(target_item_dir): if not options.upgrade: logger.warning( From 7f5fd780850901c05c7bb2d2773213d82524f3f7 Mon Sep 17 00:00:00 2001 From: Valentin Haenel Date: Fri, 12 Feb 2016 09:36:25 +0100 Subject: [PATCH 5/6] more fix --- pip/commands/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index 94b0acb5917..fd6892bf807 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -359,6 +359,7 @@ def run(self, options, args): if os.path.exists(lib_dir_platlib): self._install_to_target(lib_dir_platlib, options.target_dir) + shutil.rmtree(temp_target_dir) return requirement_set @staticmethod @@ -391,4 +392,3 @@ def _install_to_target(lib_dir, target_dir): os.path.join(lib_dir, item), target_item_dir ) - shutil.rmtree(temp_target_dir) From 154bacbeac907bd8756547a26f518b0f54a9a5a0 Mon Sep 17 00:00:00 2001 From: Valentin Haenel Date: Fri, 12 Feb 2016 11:23:26 +0100 Subject: [PATCH 6/6] moar fixes --- pip/commands/install.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pip/commands/install.py b/pip/commands/install.py index fd6892bf807..12aabcd0b37 100644 --- a/pip/commands/install.py +++ b/pip/commands/install.py @@ -355,15 +355,17 @@ def run(self, options, args): lib_dir_purelib = distutils_scheme('', home=temp_target_dir)['purelib'] lib_dir_platlib = distutils_scheme('', home=temp_target_dir)['platlib'] if os.path.exists(lib_dir_purelib): - self._install_to_target(lib_dir_purelib, options.target_dir) + self._install_to_target(lib_dir_purelib, options.target_dir, + options) if os.path.exists(lib_dir_platlib): - self._install_to_target(lib_dir_platlib, options.target_dir) + self._install_to_target(lib_dir_platlib, options.target_dir, + options) shutil.rmtree(temp_target_dir) return requirement_set @staticmethod - def _install_to_target(lib_dir, target_dir): + def _install_to_target(lib_dir, target_dir, options): for item in os.listdir(lib_dir): target_item_dir = os.path.join(target_dir, item) if os.path.exists(target_item_dir):