From 10a1e745ef0c1c00e44c7243a627b6aadaba4dfd Mon Sep 17 00:00:00 2001 From: scosman Date: Wed, 19 Mar 2025 17:05:41 -0400 Subject: [PATCH] Move pyarrow to an in method import. This allows client to exclude the pyarrow dep if they don't need it. Saved ~80MB and more compatible with older systems. Will still get a runtime error if they exclude it, then try to use it. Still works as expected unless users go out of their way to manually exclude this dependency (I'm not removing the dep, you need to manually exclude it). --- src/together/utils/files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/together/utils/files.py b/src/together/utils/files.py index e1e1d4e..77d2909 100644 --- a/src/together/utils/files.py +++ b/src/together/utils/files.py @@ -6,7 +6,6 @@ from traceback import format_exc from typing import Any, Dict, List -from pyarrow import ArrowInvalid, parquet from together.constants import ( MAX_FILE_SIZE_GB, @@ -372,6 +371,8 @@ def _check_jsonl(file: Path) -> Dict[str, Any]: def _check_parquet(file: Path) -> Dict[str, Any]: + # in method import - this allows client to exclude the pyarrow dep if they don't need it. Saved ~80MB and more compatible with older systems. + from pyarrow import ArrowInvalid, parquet report_dict: Dict[str, Any] = {} try: