CI: Test on Python 3.8, drop 3.5, format with Black, lint with F… (#176)

CI: Test on Python 3.8, drop 3.5, format with Black, lint with Flake8

Author: hugovk

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 88

.travis.yml

@@ -4,9 +4,9 @@ cache: pip
 matrix:
   fast_finish: true
   include:
+    - python: 3.8
     - python: 3.7
     - python: 3.6
-    - python: 3.5
 
 # whitelist
 # gh-pages is otherwise ignored by Travis CI
@@ -18,12 +18,13 @@ branches:
 
 install:
   - pip install -r requirements.txt
-  - pip install coveralls pyflakes
+  - pip install black coveralls flake8
 
 script:
   - coverage run  --append --source scripts test/test_gpo_member_photos.py
   - coverage run  --append --source scripts scripts/missing.py
-  - pyflakes scripts/*.py
+  - black --check --diff .
+  - flake8 scripts
 
 after_success:
   - coverage report

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.black]
+target_version = ['py36']

scripts/gpo_member_photos.py

@@ -17,12 +17,15 @@
 # pip install -r requirements.txt
 import mechanicalsoup
 
-USER_AGENT = ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 '
-              '(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36')
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
+)
 
 regex1 = re.compile(
     r'<a href="https://www.congress.gov/member/[^/]+/(\w+)[^<]+</a></span>'
-    '[^<]*<div[^<]+<div class="member-image"><img src="/img/member/([^\"]+)"')
+    '[^<]*<div[^<]+<div class="member-image"><img src="/img/member/([^"]+)"'
+)
 
 regex2 = re.compile('<a class="next" href="([^"]+)">')
 
@@ -49,18 +52,20 @@ def get_photo_list(br, congress_number, delay):
         # Fetch a page of results from Congress.gov.
         print("Page %d of Congress.gov Member listing..." % page)
         response = br.get(
-            "https://www.congress.gov/search?" +
-            urlencode({
-                "q": json.dumps(
-                    {"source": "members",
-                     "congress": str(congress_number)}),
-                "pageSize": 250,
-                "page": page,
-                })).text
+            "https://www.congress.gov/search?"
+            + urlencode(
+                {
+                    "q": json.dumps(
+                        {"source": "members", "congress": str(congress_number)}
+                    ),
+                    "pageSize": 250,
+                    "page": page,
+                }
+            )
+        ).text
 
         if len(response) == 0:
-            sys.exit("Page is blank. Try again later, you may have hit a "
-                     "limit.")
+            sys.exit("Page is blank. Try again later, you may have hit a limit.")
 
         # Scan for links to Member pages and img tags. The link to the
         # Congress.gov page uses the Member's Bioguide ID as the key, and the
@@ -116,8 +121,7 @@ def download_photos(br, photo_list, outdir, delay):
     ok = 0
 
     for bioguide_id, photo_filename in photo_list:
-        photo_url = ("https://memberguide.gpo.gov/PictorialImages/" +
-                     photo_filename)
+        photo_url = "https://memberguide.gpo.gov/PictorialImages/" + photo_filename
         print(bioguide_id, photo_url)
 
         filename = os.path.join(outdir, bioguide_id + ".jpg")
@@ -145,20 +149,35 @@ def resize_photos():
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Scrape https://memberguide.gpo.gov and save "
-                    "members' photos named after their Bioguide IDs",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        "members' photos named after their Bioguide IDs",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
     parser.add_argument(
-        '-n', '--congress', default='114',
-        help="Congress session number, for example: 110, 111, 112, 113")
+        "-n",
+        "--congress",
+        default="114",
+        help="Congress session number, for example: 110, 111, 112, 113",
+    )
     parser.add_argument(
-        '-o', '--outdir', default="congress/original",
-        help="Directory to save photos in")
+        "-o",
+        "--outdir",
+        default="congress/original",
+        help="Directory to save photos in",
+    )
     parser.add_argument(
-        '-d', '--delay', type=int, default=5, metavar='seconds',
-        help="Rate-limiting delay between scrape requests")
+        "-d",
+        "--delay",
+        type=int,
+        default=5,
+        metavar="seconds",
+        help="Rate-limiting delay between scrape requests",
+    )
     parser.add_argument(
-        '-t', '--test', action='store_true',
-        help="Test mode: don't actually save images")
+        "-t",
+        "--test",
+        action="store_true",
+        help="Test mode: don't actually save images",
+    )
     args = parser.parse_args()
 
     br = mechanicalsoup.Browser()

scripts/missing.py

@@ -13,16 +13,20 @@ def download_legislator_data():
     # clone it if it's not out
     if not os.path.exists("congress-legislators"):
         print("Cloning the congress-legislators repo...")
-        os.system("git clone -q --depth 1 "
-                  "https://github.com/unitedstates/congress-legislators "
-                  "congress-legislators")
+        os.system(
+            "git clone -q --depth 1 "
+            "https://github.com/unitedstates/congress-legislators "
+            "congress-legislators"
+        )
 
     # Update the repo so we have the latest.
     print("Updating the congress-legislators repo...")
     # these two == git pull, but git pull ignores -q on the merge part
     # so is less quiet
-    os.system("cd congress-legislators; git fetch -pq; "
-              "git merge --ff-only -q origin/master")
+    os.system(
+        "cd congress-legislators; git fetch -pq; "
+        "git merge --ff-only -q origin/master"
+    )
 
 
 def load_yaml(filename):
@@ -36,7 +40,7 @@ def file_exists(filename):
     if not os.path.exists(filename):
         print("---")
         print("Not found:", filename)
-        print(l['name'])
+        print(l["name"])
         return False
     return True
 
@@ -45,10 +49,9 @@ def file_exists(filename):
     # clone or update legislator YAML
     download_legislator_data()
 
-    legislators = load_yaml(
-        "congress-legislators/legislators-current.yaml")
+    legislators = load_yaml("congress-legislators/legislators-current.yaml")
     for l in legislators:
-        bioguide = l['id']['bioguide']
+        bioguide = l["id"]["bioguide"]
         filename = os.path.join("congress", "original", bioguide + ".jpg")
         if file_exists(filename):
             # Only check for yaml if jpg exists

test/test_gpo_member_photos.py

@@ -9,12 +9,11 @@
 import sys
 import unittest
 
-sys.path.insert(0, 'scripts')
+sys.path.insert(0, "scripts")
 import gpo_member_photos
 
 
 class TestSequenceFunctions(unittest.TestCase):
-
     def test_save_metadata(self):
         """ Test file is saved """
         bioguide_id = "A000000"
@@ -40,11 +39,11 @@ def test_pause(self):
         time2 = datetime.datetime.now()
 
         # Assert
-        self.assertLess(time1-time0, delta)
-        self.assertGreaterEqual(time2-time1, delta)
+        self.assertLess(time1 - time0, delta)
+        self.assertGreaterEqual(time2 - time1, delta)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()
 
 # End of file