sequential, threaded & async HTTP clients using HTTPX

Author: ramalho

20-futures/getflags/README.adoc

@@ -14,7 +14,7 @@ Contents:
 * <<macos_certificates>>
 
 [[server_setup]]
-== Setting up a test server
+== Setting up test servers
 
 If you don't already have a local HTTP server for testing,
 here are the steps to experiment with the `flags2*` examples
@@ -25,7 +25,7 @@ using just the Python ≥ 3.9 distribution:
 . Unzip the _flags.zip_ file, creating a _flags_ directory at _20-futures/getflags/flags/_.
 . Open a second shell, go to the _20-futures/getflags/_ directory and run `python3 -m http.server`. This will start a `ThreadingHTTPServer` listening to port 8000, serving the local files. If you open the URL http://localhost:8000/flags/[http://localhost:8000/flags/] with your browser, you'll see a long list of directories named with two-letter country codes from `ad/` to `zw/`.
 . Now you can go back to the first shell and run the _flags2*.py_ examples with the default `--server LOCAL` option.
-. To test with the `--server DELAY` option, go to _20-futures/getflags/_ and run `python3 slow_server.py`. This binds to port 8001 by default. It will add a .5s delay before each response.
+. To test with the `--server DELAY` option, go to _20-futures/getflags/_ and run `python3 slow_server.py`. This binds to port 8001 by default. It will add a random delay of .5s to 5s before each response.
 . To test with the `--server ERROR` option, go to _20-futures/getflags/_ and run `python3 slow_server.py 8002 --error-rate .25`.
 Each request will have a 25% probability of getting a
 https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/418[418 I'm a teapot] response,
@@ -86,7 +86,7 @@ optional arguments:
 All arguments are optional. The most important arguments are discussed next.
 
 One option you can't ignore is `-s/--server`: it lets you choose which HTTP server and base URL will be used in the test.
-You can pass one of four strings to determine where the script will look for the flags (the strings are case insensitive):
+You can pass one of four labels to determine where the script will look for the flags (the labels are case-insensitive):
 
 `LOCAL`:: Use `http://localhost:8000/flags`; this is the default.
 You should configure a local HTTP server to answer at port 8000. See <<server_setup>> for instructions.

20-futures/getflags/flags.py

@@ -21,35 +21,38 @@
 from pathlib import Path
 from typing import Callable
 
-import requests  # <1>
+import httpx  # <1>
 
 POP20_CC = ('CN IN US ID BR PK NG BD RU JP '
             'MX PH VN ET EG DE IR TR CD FR').split()  # <2>
 
-BASE_URL = 'http://fluentpython.com/data/flags'       # <3>
+BASE_URL = 'https://www.fluentpython.com/data/flags'  # <3>
 DEST_DIR = Path('downloaded')                         # <4>
 
 def save_flag(img: bytes, filename: str) -> None:     # <5>
     (DEST_DIR / filename).write_bytes(img)
 
 def get_flag(cc: str) -> bytes:  # <6>
     url = f'{BASE_URL}/{cc}/{cc}.gif'.lower()
-    resp = requests.get(url)
+    resp = httpx.get(url, timeout=6.1,       # <7>
+                     follow_redirects=True)  # <8>
+    resp.raise_for_status()  # <9>
     return resp.content
 
-def download_many(cc_list: list[str]) -> int:  # <7>
-    for cc in sorted(cc_list):                 # <8>
+def download_many(cc_list: list[str]) -> int:  # <10>
+    for cc in sorted(cc_list):                 # <11>
         image = get_flag(cc)
         save_flag(image, f'{cc}.gif')
-        print(cc, end=' ', flush=True)         # <9>
+        print(cc, end=' ', flush=True)         # <12>
     return len(cc_list)
 
-def main(downloader: Callable[[list[str]], int]) -> None:  # <10>
-    t0 = time.perf_counter()                               # <11>
+def main(downloader: Callable[[list[str]], int]) -> None:  # <13>
+    DEST_DIR.mkdir(exist_ok=True)                          # <14>
+    t0 = time.perf_counter()                               # <15>
     count = downloader(POP20_CC)
     elapsed = time.perf_counter() - t0
     print(f'\n{count} downloads in {elapsed:.2f}s')
 
 if __name__ == '__main__':
-    main(download_many)     # <12>
+    main(download_many)     # <16>
 # end::FLAGS_PY[]

20-futures/getflags/flags2_asyncio.py

@@ -8,91 +8,94 @@
 # tag::FLAGS2_ASYNCIO_TOP[]
 import asyncio
 from collections import Counter
+from http import HTTPStatus
+from pathlib import Path
 
-import aiohttp
+import httpx
 import tqdm  # type: ignore
 
-from flags2_common import main, HTTPStatus, Result, save_flag
+from flags2_common import main, DownloadStatus, save_flag
 
 # default set low to avoid errors from remote site, such as
 # 503 - Service Temporarily Unavailable
 DEFAULT_CONCUR_REQ = 5
 MAX_CONCUR_REQ = 1000
 
-
-class FetchError(Exception):  # <1>
-    def __init__(self, country_code: str):
-        self.country_code = country_code
-
-
-async def get_flag(session: aiohttp.ClientSession,  # <2>
+async def get_flag(session: httpx.AsyncClient,  # <2>
                    base_url: str,
                    cc: str) -> bytes:
     url = f'{base_url}/{cc}/{cc}.gif'.lower()
-    async with session.get(url) as resp:
-        if resp.status == 200:
-            return await resp.read()
-        else:
-            resp.raise_for_status()  # <3>
-            return bytes()
+    resp = await session.get(url, timeout=3.1, follow_redirects=True)   # <3>
+    resp.raise_for_status()
+    return resp.content
 
-async def download_one(session: aiohttp.ClientSession,
+async def download_one(session: httpx.AsyncClient,
                        cc: str,
                        base_url: str,
                        semaphore: asyncio.Semaphore,  # <4>
-                       verbose: bool) -> Result:
+                       verbose: bool) -> DownloadStatus:
     try:
         async with semaphore:  # <5>
             image = await get_flag(session, base_url, cc)
-    except aiohttp.ClientResponseError as exc:
-        if exc.status == 404:               # <6>
-            status = HTTPStatus.not_found
-            msg = 'not found'
+    except httpx.HTTPStatusError as exc:  # <4>
+        res = exc.response
+        if res.status_code == HTTPStatus.NOT_FOUND:
+            status = DownloadStatus.NOT_FOUND  # <5>
+            msg = f'not found: {res.url}'
         else:
-            raise FetchError(cc) from exc  # <7>
+            raise
+
     else:
-        save_flag(image, f'{cc}.gif')
-        status = HTTPStatus.ok
+        await asyncio.to_thread(save_flag, image, f'{cc}.gif')
+        status = DownloadStatus.OK
         msg = 'OK'
     if verbose and msg:
         print(cc, msg)
-    return Result(status, cc)
+    return status
 # end::FLAGS2_ASYNCIO_TOP[]
 
 # tag::FLAGS2_ASYNCIO_START[]
 async def supervisor(cc_list: list[str],
                      base_url: str,
                      verbose: bool,
-                     concur_req: int) -> Counter[HTTPStatus]:  # <1>
-    counter: Counter[HTTPStatus] = Counter()
+                     concur_req: int) -> Counter[DownloadStatus]:  # <1>
+    counter: Counter[DownloadStatus] = Counter()
     semaphore = asyncio.Semaphore(concur_req)  # <2>
-    async with aiohttp.ClientSession() as session:
+    async with httpx.AsyncClient() as session:
         to_do = [download_one(session, cc, base_url, semaphore, verbose)
                  for cc in sorted(cc_list)]  # <3>
         to_do_iter = asyncio.as_completed(to_do)  # <4>
         if not verbose:
             to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))  # <5>
         for coro in to_do_iter:  # <6>
             try:
-                res = await coro  # <7>
-            except FetchError as exc:  # <8>
-                country_code = exc.country_code  # <9>
-                try:
-                    error_msg = exc.__cause__.message  # type: ignore  # <10>
-                except AttributeError:
-                    error_msg = 'Unknown cause'  # <11>
-                if verbose and error_msg:
-                    print(f'*** Error for {country_code}: {error_msg}')
-                status = HTTPStatus.error
-            else:
-                status = res.status
-            counter[status] += 1  # <12>
-    return counter  # <13>
+                status = await coro  # <7>
+            except httpx.HTTPStatusError as exc:  # <8>
+                error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}'
+                error_msg = error_msg.format(resp=exc.response)
+                error = exc
+            except httpx.RequestError as exc:  # <9>
+                error_msg = f'{exc} {type(exc)}'.strip()
+                error = exc
+            except KeyboardInterrupt:  # <10>
+                break
+            else:  # <11>
+                error = None
+
+            if error:
+                status = DownloadStatus.ERROR  # <12>
+                if verbose:
+                    url = str(error.request.url)  # <13>
+                    cc = Path(url).stem.upper()   # <14>
+                    print(f'{cc} error: {error_msg}')
+            counter[status] += 1
+
+    return counter
 
 def download_many(cc_list: list[str],
                   base_url: str,
                   verbose: bool,
-                  concur_req: int) -> Counter[HTTPStatus]:
+                  concur_req: int) -> Counter[DownloadStatus]:
     coro = supervisor(cc_list, base_url, verbose, concur_req)
     counts = asyncio.run(coro)  # <14>
 

20-futures/getflags/flags2_asyncio_executor.py

@@ -2,108 +2,107 @@
 
 """Download flags of countries (with error handling).
 
-asyncio async/await version using run_in_executor for save_flag.
+asyncio async/await version
 
 """
-
+# tag::FLAGS2_ASYNCIO_TOP[]
 import asyncio
 from collections import Counter
+from http import HTTPStatus
+from pathlib import Path
 
-import aiohttp
+import httpx
 import tqdm  # type: ignore
 
-from flags2_common import main, HTTPStatus, Result, save_flag
+from flags2_common import main, DownloadStatus, save_flag
 
 # default set low to avoid errors from remote site, such as
 # 503 - Service Temporarily Unavailable
 DEFAULT_CONCUR_REQ = 5
 MAX_CONCUR_REQ = 1000
 
 
-class FetchError(Exception):
-    def __init__(self, country_code: str):
-        self.country_code = country_code
-
-
-async def get_flag(session: aiohttp.ClientSession,
+async def get_flag(session: httpx.AsyncClient,  # <2>
                    base_url: str,
                    cc: str) -> bytes:
     url = f'{base_url}/{cc}/{cc}.gif'.lower()
-    async with session.get(url) as resp:
-        if resp.status == 200:
-            return await resp.read()
-        else:
-            resp.raise_for_status()
-            return bytes()
+    resp = await session.get(url, timeout=3.1, follow_redirects=True)   # <3>
+    resp.raise_for_status()
+    return resp.content
 
-# tag::FLAGS2_ASYNCIO_EXECUTOR[]
-async def download_one(session: aiohttp.ClientSession,
+
+async def download_one(session: httpx.AsyncClient,
                        cc: str,
                        base_url: str,
                        semaphore: asyncio.Semaphore,
-                       verbose: bool) -> Result:
+                       verbose: bool) -> DownloadStatus:
     try:
         async with semaphore:
             image = await get_flag(session, base_url, cc)
-    except aiohttp.ClientResponseError as exc:
-        if exc.status == 404:
-            status = HTTPStatus.not_found
-            msg = 'not found'
+    except httpx.HTTPStatusError as exc:
+        res = exc.response
+        if res.status_code == HTTPStatus.NOT_FOUND:
+            status = DownloadStatus.NOT_FOUND
+            msg = f'not found: {res.url}'
         else:
-            raise FetchError(cc) from exc
+            raise
     else:
-        loop = asyncio.get_running_loop()   # <1>
-        loop.run_in_executor(None,          # <2>
-            save_flag, image, f'{cc}.gif')  # <3>
-        status = HTTPStatus.ok
+# tag::FLAGS2_ASYNCIO_EXECUTOR[]
+        loop = asyncio.get_running_loop()         # <1>
+        loop.run_in_executor(None, save_flag,     # <2>
+                             image, f'{cc}.gif')  # <3>
+# end::FLAGS2_ASYNCIO_EXECUTOR[]
+        status = DownloadStatus.OK
         msg = 'OK'
     if verbose and msg:
         print(cc, msg)
-    return Result(status, cc)
-# end::FLAGS2_ASYNCIO_EXECUTOR[]
+    return status
 
 async def supervisor(cc_list: list[str],
                      base_url: str,
                      verbose: bool,
-                     concur_req: int) -> Counter[HTTPStatus]:
-    counter: Counter[HTTPStatus] = Counter()
-    semaphore = asyncio.Semaphore(concur_req)
-    async with aiohttp.ClientSession() as session:
+                     concur_req: int) -> Counter[DownloadStatus]:  # <1>
+    counter: Counter[DownloadStatus] = Counter()
+    semaphore = asyncio.Semaphore(concur_req)  # <2>
+    async with httpx.AsyncClient() as session:
         to_do = [download_one(session, cc, base_url, semaphore, verbose)
-                 for cc in sorted(cc_list)]
-
-        to_do_iter = asyncio.as_completed(to_do)
+                 for cc in sorted(cc_list)]  # <3>
+        to_do_iter = asyncio.as_completed(to_do)  # <4>
         if not verbose:
-            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))
-        for coro in to_do_iter:
+            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))  # <5>
+        for coro in to_do_iter:  # <6>
             try:
-                res = await coro
-            except FetchError as exc:
-                country_code = exc.country_code
-                try:
-                    error_msg = exc.__cause__.message  # type: ignore
-                except AttributeError:
-                    error_msg = 'Unknown cause'
-                if verbose and error_msg:
-                    print(f'*** Error for {country_code}: {error_msg}')
-                status = HTTPStatus.error
-            else:
-                status = res.status
-
-            counter[status] += 1
-
-    return counter
-
+                status = await coro  # <7>
+            except httpx.HTTPStatusError as exc:  # <13>
+                error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}'
+                error_msg = error_msg.format(resp=exc.response)
+                error = exc
+            except httpx.RequestError as exc:  # <15>
+                error_msg = f'{exc} {type(exc)}'.strip()
+                error = exc
+            except KeyboardInterrupt:  # <7>
+                break
+            else:  # <8>
+                error = None
+
+            if error:
+                status = DownloadStatus.ERROR  # <9>
+                if verbose:      # <11>
+                    cc = Path(str(error.request.url)).stem.upper()
+                    print(f'{cc} error: {error_msg}')
+            counter[status] += 1           # <10>
+
+    return counter  # <12>
 
 def download_many(cc_list: list[str],
                   base_url: str,
                   verbose: bool,
-                  concur_req: int) -> Counter[HTTPStatus]:
+                  concur_req: int) -> Counter[DownloadStatus]:
     coro = supervisor(cc_list, base_url, verbose, concur_req)
     counts = asyncio.run(coro)  # <14>
 
     return counts
 
-
 if __name__ == '__main__':
     main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)
+# end::FLAGS2_ASYNCIO_START[]