From bf57430e4a81b0018a078ac103d552b435c2c246 Mon Sep 17 00:00:00 2001 From: Sakari Ikonen Date: Mon, 26 Aug 2024 22:49:59 +0300 Subject: [PATCH] deal with empty string and None values in s3 endpoint_url --- metaflow/metaflow_environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metaflow/metaflow_environment.py b/metaflow/metaflow_environment.py index e180ab9bb0b..33ffe02dcfa 100644 --- a/metaflow/metaflow_environment.py +++ b/metaflow/metaflow_environment.py @@ -94,9 +94,11 @@ def _get_download_code_package_cmd(self, code_package_url, datastore_type): bucket, s3_object = parse_s3_full_path(code_package_url) # NOTE: the script quoting is extremely sensitive due to the way shlex.split operates and this being inserted # into a quoted command elsewhere. + # NOTE: Reason for the extra conditionals in the script are because + # Boto3 does not play well with passing None or an empty string to endpoint_url return "{python} -c '{script}'".format( python=self._python(), - script='import boto3, os; boto3.client(\\"s3\\", endpoint_url=os.getenv(\\"METAFLOW_S3_ENDPOINT_URL\\")).download_file(\\"%s\\", \\"%s\\", \\"job.tar\\")' + script='import boto3, os; ep=os.getenv(\\"METAFLOW_S3_ENDPOINT_URL\\"; (boto3.client(\\"s3\\", endpoint_url=ep) if ep else boto3.client(\\"s3\\")).download_file(\\"%s\\", \\"%s\\", \\"job.tar\\")' % (bucket, s3_object), ) elif datastore_type == "azure":