Run url tests on real urls on one runner (3.14 on Ubuntu)

Try usgs.gov instead of github.com for doctests

Put https://github.com/nvkelso/natural-earth-vector tests in a separate localhost only test function

Avoid nvkelson's repo (natural-earth-vector)

don't do doctests

Try true / false instead of yes / no

Try to test remote urls from one job in the test matrix only

Add extra url of .zipped shapefile and test it

Author: JamesParrott

.github/actions/test/action.yml

@@ -10,8 +10,8 @@ inputs:
     default: '-m "not network"'
     required: false
   replace_remote_urls_with_localhost:
-    description: yes or no.  Test loading shapefiles from a url, without overloading an external server from 30 parallel workflows.
-    default: 'no'
+    description: true or false.  Test loading shapefiles from a url, without overloading an external server from 30 parallel workflows.
+    default: 'false'
     required: false
   pyshp_repo_directory:
     description: Path to where the PyShp repo was checked out to (to keep separate from Shapefiles & artefacts repo).
@@ -40,7 +40,7 @@ runs:
     #   uses: ./Pyshp/.github/actions/test
     #   with:
     #     extra_args: ""
-    #     replace_remote_urls_with_localhost: 'yes'
+    #     replace_remote_urls_with_localhost: 'true'
     #     pyshp_repo_directory: ./Pyshp
 
     # The Python to be tested with is required to already be setup,
@@ -51,14 +51,14 @@ runs:
 
 
     - name: Checkout shapefiles and zip file artefacts repo
-      if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' }}
+      if: ${{ inputs.replace_remote_urls_with_localhost == 'true' }}
       uses: actions/checkout@v6
       with:
         repository: JamesParrott/PyShp_test_shapefile
         path: ./PyShp_test_shapefile
 
     - name: Serve shapefiles and zip file artefacts on localhost
-      if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' }}
+      if: ${{ inputs.replace_remote_urls_with_localhost == 'true' }}
       shell: bash
       working-directory: ./PyShp_test_shapefile
       run: |
@@ -105,12 +105,12 @@ runs:
     # - name: Test http server
     #   # (needs a full Github Actions runner or a Python non-slim Docker image,
     #   # as the slim Debian images don't have curl or wget).
-    #   if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' }}
+    #   if: ${{ inputs.replace_remote_urls_with_localhost == 'true' }}
     #   shell: bash
     #   run: curl http://localhost:8000/ne_110m_admin_0_tiny_countries.shp
 
     - name: Stop http server
-      if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' }}
+      if: ${{ inputs.replace_remote_urls_with_localhost == 'true' }}
       shell: bash
       run: |
         echo Killing http server process ID: ${{ env.HTTP_SERVER_PID }}

.github/workflows/run_checks_build_and_test.yml

@@ -87,10 +87,10 @@ jobs:
       with:
         pyshp_repo_directory: ./Pyshp
 
-    - name: Network tests
+    - name: "Network tests (on Localhost ${{ !(matrix.os == 'ubuntu-24.04' && matrix.python-version == '3.14') }})"
       uses: ./Pyshp/.github/actions/test
       with:
         extra_args: '-m network'
-        replace_remote_urls_with_localhost: 'yes'
+        replace_remote_urls_with_localhost: ${{ !(matrix.os == 'ubuntu-24.04' && matrix.python-version == '3.14') }}
         # Checkout to ./PyShp, as the test job also needs to check out the artefact repo
         pyshp_repo_directory: ./Pyshp

README.md

@@ -435,7 +435,7 @@ Finally, you can use all of the above methods to read shapefiles directly from t
 	>>> sf = shapefile.Reader("https://github.com/JamesParrott/PyShp_test_shapefile/raw/main/gis_osm_natural_a_free_1.zip")
 
 	>>> # from a shapefile collection of files in a github repository
-	>>> sf = shapefile.Reader("https://github.com/nvkelso/natural-earth-vector/blob/master/110m_cultural/ne_110m_admin_0_tiny_countries.shp?raw=true")
+	>>> sf = shapefile.Reader("https://pubs.usgs.gov/of/2000/of00-006/gisdata/coverage/airports.shp")
 
 This will automatically download the file(s) to a temporary location before reading, saving you a lot of time and repetitive boilerplate code when you just want quick access to some external data.
 

src/shapefile.py

@@ -60,7 +60,7 @@
 
 # Test config (for the Doctest runner and test_shapefile.py)
 REPLACE_REMOTE_URLS_WITH_LOCALHOST = (
-    os.getenv("REPLACE_REMOTE_URLS_WITH_LOCALHOST", "").lower() == "yes"
+    os.getenv("REPLACE_REMOTE_URLS_WITH_LOCALHOST", "").lower() == "true"
 )
 
 # Constants for shape types

test_shapefile.py

@@ -479,25 +479,25 @@ def test_shaperecord_geo_interface():
             assert json.dumps(shaperec.__geo_interface__)
 
 
+@pytest.mark.skipif(
+    not shapefile.REPLACE_REMOTE_URLS_WITH_LOCALHOST,
+    reason="Flakey test, fails due to Github rate limit",
+)
 @pytest.mark.network
-def test_reader_url():
+def test_reader_nvkelso_files_from_localhost_url():
     """
     Assert that Reader can open shapefiles from a url.
     """
 
-    # Allow testing loading of shapefiles from a url on localhost (to avoid
-    # overloading external servers, and associated spurious test failures).
+    # Only test these shapefiles from localhost,
+    # https://github.com/nvkelso/natural-earth-vector urls throws 426 errors ("too many downloads").
     # A suitable repo of test files, and a localhost server setup is
     # defined in ./.github/actions/test/actions.yml
-    if shapefile.REPLACE_REMOTE_URLS_WITH_LOCALHOST:
 
-        def Reader(url):
-            new_url = shapefile._replace_remote_url(url)
-            print(f"repr(new_url): {repr(new_url)}")
-            return shapefile.Reader(new_url)
-    else:
-        print("Using plain Reader")
-        Reader = shapefile.Reader
+    def Reader(url):
+        new_url = shapefile._replace_remote_url(url)
+        print(f"repr(new_url): {repr(new_url)}")
+        return shapefile.Reader(new_url)
 
     # test with extension
     url = "https://github.com/nvkelso/natural-earth-vector/blob/master/110m_cultural/ne_110m_admin_0_tiny_countries.shp?raw=true"
@@ -518,19 +518,57 @@ def Reader(url):
     assert sf._shx is None or sf.shx.closed
     assert sf.dbf.closed
 
+
+@pytest.mark.network
+def test_reader_urls():
+    """
+    Assert that Reader can open shapefiles from a few different urls.
+    """
+
+    # Allow testing loading of shapefiles from a url on localhost (to avoid
+    # overloading external servers, and associated spurious test failures).
+    # A suitable repo of test files, and a localhost server setup is
+    # defined in ./.github/actions/test/actions.yml
+    if shapefile.REPLACE_REMOTE_URLS_WITH_LOCALHOST:
+
+        def Reader(url):
+            new_url = shapefile._replace_remote_url(url)
+            print(f"repr(new_url): {repr(new_url)}")
+            return shapefile.Reader(new_url)
+    else:
+        print("Using plain Reader")
+        Reader = shapefile.Reader
+
     # test no files found
     url = "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/README.md"
     with pytest.raises(shapefile.ShapefileException):
         with Reader(url) as sf:
             pass
 
     # test reading zipfile from url
-    url = "https://github.com/JamesParrott/PyShp_test_shapefile/raw/main/gis_osm_natural_a_free_1.zip"
-    with Reader(url) as sf:
-        for __recShape in sf.iterShapeRecords():
+    urls = [
+        "https://github.com/JamesParrott/PyShp_test_shapefile/raw/main/gis_osm_natural_a_free_1.zip",
+        "http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip",
+    ]
+    for url in urls:
+        try:
+            with Reader(url) as sf:
+                for __recShape in sf.iterShapeRecords():
+                    pass
+                assert len(sf) > 0
+        except shapefile.HTTPError:
             pass
-        assert len(sf) > 0
-    assert sf.shp.closed is sf.shx.closed is sf.dbf.closed is True
+        else:
+            assert sf.shp.closed is sf.shx.closed is sf.dbf.closed is True
+            break
+    else:
+        raise shapefile.HTTPError(
+            "\n".join(urls),
+            "Could not download .zipped shapefiles from any of the test urls",
+            404,
+            {},
+            None,
+        )
 
 
 def test_reader_zip():