From 5e6b15799501633c9176a9f76527b47fe38f698e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Dec 2019 18:11:27 -0800 Subject: [PATCH 1/2] CLN: use stdlib Iterator instead of BaseIterator --- pandas/io/common.py | 19 +++++-------------- pandas/io/json/_json.py | 4 ++-- pandas/io/parsers.py | 6 +++--- pandas/io/sas/sas7bdat.py | 5 +++-- pandas/io/sas/sas_xport.py | 6 +++--- pandas/io/stata.py | 6 +++--- 6 files changed, 19 insertions(+), 27 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index a01011cd7d4e4..ed6d7d58d2671 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -2,6 +2,7 @@ import bz2 import codecs +from collections.abc import Iterator import csv import gzip from io import BufferedIOBase, BytesIO @@ -75,18 +76,6 @@ _VALID_URLS.discard("") -class BaseIterator: - """Subclass this and provide a "__next__()" method to obtain an iterator. - Useful only when the object being iterated is non-reusable (e.g. OK for a - parser, not for an in-memory table, yes for its iterator).""" - - def __iter__(self) -> "BaseIterator": - return self - - def __next__(self): - raise AbstractMethodError(self) - - def _is_url(url) -> bool: """ Check to see if a URL has a valid protocol. @@ -541,7 +530,7 @@ def closed(self): return self.fp is None -class MMapWrapper(BaseIterator): +class MMapWrapper(Iterator): """ Wrapper for the Python's mmap class so that it can be properly read in by Python's csv.reader class. @@ -578,7 +567,7 @@ def __next__(self) -> str: return newline -class UTF8Recoder(BaseIterator): +class UTF8Recoder(Iterator): """ Iterator that reads an encoded stream and re-encodes the input to UTF-8 """ @@ -595,6 +584,8 @@ def readline(self) -> bytes: def next(self) -> bytes: return next(self.reader).encode("utf-8") + __next__ = next + def close(self): self.reader.close() diff --git a/pandas/io/json/_json.py b/pandas/io/json/_json.py index 6cb811bb97755..fc6dbe0c907a6 100644 --- a/pandas/io/json/_json.py +++ b/pandas/io/json/_json.py @@ -1,4 +1,5 @@ from collections import OrderedDict +from collections.abc import Iterator import functools from io import StringIO from itertools import islice @@ -19,7 +20,6 @@ from pandas.core.reshape.concat import concat from pandas.io.common import ( - BaseIterator, _get_handle, _infer_compression, _stringify_path, @@ -616,7 +616,7 @@ def read_json( return result -class JsonReader(BaseIterator): +class JsonReader(Iterator): """ JsonReader provides an interface for reading in a JSON file. diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py index a887a537a2201..1b8381f95ada5 100755 --- a/pandas/io/parsers.py +++ b/pandas/io/parsers.py @@ -3,6 +3,7 @@ """ from collections import defaultdict +from collections.abc import Iterator import csv import datetime from io import StringIO @@ -61,7 +62,6 @@ from pandas.io.common import ( _NA_VALUES, - BaseIterator, UnicodeReader, UTF8Recoder, _get_handle, @@ -787,7 +787,7 @@ def read_fwf( return _read(filepath_or_buffer, kwds) -class TextFileReader(BaseIterator): +class TextFileReader(Iterator): """ Passed dialect overrides any of the related parser options @@ -3593,7 +3593,7 @@ def _get_col_names(colspec, columns): return colnames -class FixedWidthReader(BaseIterator): +class FixedWidthReader(Iterator): """ A reader of fixed-width lines. """ diff --git a/pandas/io/sas/sas7bdat.py b/pandas/io/sas/sas7bdat.py index eb57d703cd4d5..fe96b94e368e3 100644 --- a/pandas/io/sas/sas7bdat.py +++ b/pandas/io/sas/sas7bdat.py @@ -13,6 +13,7 @@ Reference for binary data compression: http://collaboration.cmc.ec.gc.ca/science/rpn/biblio/ddj/Website/articles/CUJ/1992/9210/ross/ross.htm """ +from collections.abc import Iterator from datetime import datetime import struct @@ -22,7 +23,7 @@ import pandas as pd -from pandas.io.common import BaseIterator, get_filepath_or_buffer +from pandas.io.common import get_filepath_or_buffer from pandas.io.sas._sas import Parser import pandas.io.sas.sas_constants as const @@ -36,7 +37,7 @@ class _column: # SAS7BDAT represents a SAS data file in SAS7BDAT format. -class SAS7BDATReader(BaseIterator): +class SAS7BDATReader(Iterator): """ Read SAS files in SAS7BDAT format. diff --git a/pandas/io/sas/sas_xport.py b/pandas/io/sas/sas_xport.py index 9aa8ed1dfeb5d..ccaee56383a5f 100644 --- a/pandas/io/sas/sas_xport.py +++ b/pandas/io/sas/sas_xport.py @@ -7,7 +7,7 @@ https://support.sas.com/techsup/technote/ts140.pdf """ - +from collections.abc import Iterator from datetime import datetime from io import BytesIO import struct @@ -19,7 +19,7 @@ import pandas as pd -from pandas.io.common import BaseIterator, get_filepath_or_buffer +from pandas.io.common import get_filepath_or_buffer _correct_line1 = ( "HEADER RECORD*******LIBRARY HEADER RECORD!!!!!!!" @@ -251,7 +251,7 @@ def _parse_float_vec(vec): return ieee -class XportReader(BaseIterator): +class XportReader(Iterator): __doc__ = _xport_reader_doc def __init__( diff --git a/pandas/io/stata.py b/pandas/io/stata.py index dbe64e4c0f06d..527a428c5412f 100644 --- a/pandas/io/stata.py +++ b/pandas/io/stata.py @@ -9,7 +9,7 @@ You can find more information on http://presbrey.mit.edu/PyDTA and http://www.statsmodels.org/devel/ """ - +from collections.abc import Iterator import datetime from io import BytesIO import os @@ -44,7 +44,7 @@ from pandas.core.frame import DataFrame from pandas.core.series import Series -from pandas.io.common import BaseIterator, _stringify_path, get_filepath_or_buffer +from pandas.io.common import _stringify_path, get_filepath_or_buffer _version_error = ( "Version of given Stata file is not 104, 105, 108, " @@ -1010,7 +1010,7 @@ def __init__(self): ) -class StataReader(StataParser, BaseIterator): +class StataReader(StataParser, Iterator): __doc__ = _stata_reader_doc def __init__( From 3902d86a54e836040d6ede6288de8616f6b08ce9 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 19 Dec 2019 18:25:55 -0800 Subject: [PATCH 2/2] next -> __next__ --- pandas/io/common.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index ed6d7d58d2671..785139b41a60a 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -581,11 +581,9 @@ def read(self, bytes: int = -1) -> bytes: def readline(self) -> bytes: return self.reader.readline().encode("utf-8") - def next(self) -> bytes: + def __next__(self) -> bytes: return next(self.reader).encode("utf-8") - __next__ = next - def close(self): self.reader.close()