Add PURL_TYPES and enforce validation of PackageURL types

Closes #155 and #181

Author: kdeldycke

CHANGELOG.rst

@@ -4,6 +4,14 @@ Changelog
 0.18.0 (unlreleased)
 --------------------
 
+- Add ``PURL_TYPES`` constant sourced from the ``purl-types-index.json`` registry.
+  Enforce validation of PackageURL types.
+  Remove ``sourceforge`` and ``gitlab`` support from ``url2purl`` and ``purl2url``.
+  Remove ``rubygems`` route alias from ``purl2url`` (the registered type is ``gem``).
+  https://github.com/package-url/packageurl-python/issues/181
+  https://github.com/package-url/packageurl-python/issues/155
+  https://github.com/package-url/packageurl-python/pull/188
+
 - Add support for Python 3.14
 
 - Drop support for Python 3.8

src/packageurl/__init__.py

@@ -61,6 +61,56 @@
 """
 
 
+PURL_TYPES: set[str] = {
+    "alpm",
+    "apk",
+    "bazel",
+    "bitbucket",
+    "bitnami",
+    "cargo",
+    "chrome-extension",
+    "cocoapods",
+    "composer",
+    "conan",
+    "conda",
+    "cpan",
+    "cran",
+    "deb",
+    "docker",
+    "gem",
+    "generic",
+    "github",
+    "golang",
+    "hackage",
+    "hex",
+    "huggingface",
+    "julia",
+    "luarocks",
+    "maven",
+    "mlflow",
+    "npm",
+    "nuget",
+    "oci",
+    "opam",
+    "otp",
+    "pub",
+    "pypi",
+    "qpkg",
+    "rpm",
+    "swid",
+    "swift",
+    "vscode-extension",
+    "yocto",
+}
+"""Registered pURL types.
+
+Sourced from the `purl-types-index.json
+<https://github.com/package-url/purl-spec/blob/main/purl-types-index.json>`_
+registry, which is automatically regenerated whenever a new pURL type is
+registered.
+"""
+
+
 class ValidationSeverity(str, Enum):
     ERROR = "error"
     WARNING = "warning"
@@ -124,8 +174,15 @@ def normalize_type(type: AnyStr | None, encode: bool | None = True) -> str | Non
 
     type_str = type if isinstance(type, str) else type.decode("utf-8")
     quoter = get_quoter(encode)
-    type_str = quoter(type_str)
-    return type_str.strip().lower() or None
+    type_str = quoter(type_str).strip().lower()
+    if not type_str:
+        return None
+    if type_str not in PURL_TYPES:
+        raise ValueError(
+            f"Invalid purl type: {type_str!r}. "
+            f"Must be one of: {', '.join(sorted(PURL_TYPES))}."
+        )
+    return type_str
 
 
 def normalize_namespace(

src/packageurl/contrib/purl2url.py

@@ -44,7 +44,6 @@ def get_repo_download_url_by_package_type(
     download_url_by_type = {
         "github": f"https://github.com/{namespace}/{name}/archive/{version}.{archive_extension}",
         "bitbucket": f"https://bitbucket.org/{namespace}/{name}/get/{version}.{archive_extension}",
-        "gitlab": f"https://gitlab.com/{namespace}/{name}/-/archive/{version}/{name}-{version}.{archive_extension}",
     }
     return download_url_by_type.get(type)
 
@@ -158,21 +157,7 @@ def build_github_repo_url(purl):
     return repo_url
 
 
-@repo_router.route("pkg:gitlab/.*")
-def build_gitlab_repo_url(purl):
-    """
-    Return a gitlab repo URL from the `purl` string.
-    """
-    purl_data = PackageURL.from_string(purl)
-
-    namespace = purl_data.namespace
-    name = purl_data.name
-
-    if name and namespace:
-        return f"https://gitlab.com/{namespace}/{name}"
-
-
-@repo_router.route("pkg:(gem|rubygems)/.*")
+@repo_router.route("pkg:gem/.*")
 def build_rubygems_repo_url(purl):
     """
     Return a rubygems repo URL from the `purl` string.
@@ -351,7 +336,7 @@ def build_cargo_download_url(purl):
         return f"https://crates.io/api/v1/crates/{name}/{version}/download"
 
 
-@download_router.route("pkg:(gem|rubygems)/.*")
+@download_router.route("pkg:gem/.*")
 def build_rubygems_download_url(purl):
     """
     Return a rubygems download URL from the `purl` string.
@@ -435,7 +420,7 @@ def build_nuget_download_url(purl):
         return f"https://www.nuget.org/api/v2/package/{name}/{version}"
 
 
-@download_router.route("pkg:gitlab/.*", "pkg:bitbucket/.*", "pkg:github/.*")
+@download_router.route("pkg:bitbucket/.*", "pkg:github/.*")
 def build_repo_download_url(purl):
     """
     Return a gitlab download URL from the `purl` string.

src/packageurl/contrib/url2purl.py

@@ -372,75 +372,6 @@ def build_composer_purl(uri):
 register_pattern("nuget", nuget_api_pattern)
 
 
-# https://sourceforge.net/projects/turbovnc/files/3.1/turbovnc-3.1.tar.gz/download
-# https://sourceforge.net/projects/scribus/files/scribus/1.6.0/scribus-1.6.0.tar.gz/download
-# https://sourceforge.net/projects/ventoy/files/v1.0.96/Ventoy%201.0.96%20release%20source%20code.tar.gz/download
-# https://sourceforge.net/projects/geoserver/files/GeoServer/2.23.4/geoserver-2.23.4-war.zip/download
-sourceforge_download_pattern = (
-    r"^https?://.*sourceforge.net/projects/"
-    r"(?P<name>.+)/"
-    r"files/"
-    r"(?i:(?P=name)/)?"  # optional case-insensitive name segment repeated
-    r"v?(?P<version>[0-9\.]+)/"  # version restricted to digits and dots
-    r"(?i:(?P=name)).*(?P=version).*"  # case-insensitive matching for {name}-{version}
-    r"(/download)$"  # ending with "/download"
-)
-
-register_pattern("sourceforge", sourceforge_download_pattern)
-
-
-# https://sourceforge.net/projects/spacesniffer/files/spacesniffer_1_3_0_2.zip/download
-sourceforge_download_pattern_bis = (
-    r"^https?://.*sourceforge.net/projects/"
-    r"(?P<name>.+)/"
-    r"files/"
-    r"(?i:(?P=name))_*(?P<version>[0-9_]+).*"
-    r"(/download)$"  # ending with "/download"
-)
-
-register_pattern("sourceforge", sourceforge_download_pattern_bis)
-
-
-@purl_router.route("https?://.*sourceforge.net/project/.*")
-def build_sourceforge_purl(uri):
-    # We use a more general route pattern instead of using `sourceforge_pattern`
-    # below by itself because we want to capture all sourceforge download URLs,
-    # even the ones that do not fit `sourceforge_pattern`. This helps prevent
-    # url2purl from attempting to create a generic PackageURL from a sourceforge
-    # URL that we can't handle.
-
-    # http://master.dl.sourceforge.net/project/libpng/zlib/1.2.3/zlib-1.2.3.tar.bz2
-    sourceforge_pattern = (
-        r"^https?://.*sourceforge.net/projects?/"
-        r"(?P<namespace>([^/]+))/"  # do not allow more "/" segments
-        r"(OldFiles/)?"
-        r"(?P<name>.+)/"
-        r"(?P<version>[v0-9\.]+)/"  # version restricted to digits and dots
-        r"(?P=name).*(?P=version).*"  # {name}-{version} repeated in the filename
-        r"[^/]$"  # not ending with "/"
-    )
-
-    sourceforge_purl = purl_from_pattern("sourceforge", sourceforge_pattern, uri)
-
-    if not sourceforge_purl:
-        # Get the project name from `uri` and use that as the Package name
-        # http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
-        split_uri = uri.split("/project/")
-
-        # http://master.dl.sourceforge.net, aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
-        if len(split_uri) >= 2:
-            # aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
-            remaining_uri_path = split_uri[1]
-            # aloyscore, aloyscore, 0.1a1%2520stable, 0.1a1_stable_AloysCore.zip
-            remaining_uri_path_segments = remaining_uri_path.split("/")
-            if remaining_uri_path_segments:
-                project_name = remaining_uri_path_segments[0]  # aloyscore
-                sourceforge_purl = PackageURL(
-                    type="sourceforge", name=project_name, qualifiers={"download_url": uri}
-                )
-    return sourceforge_purl
-
-
 # https://crates.io/api/v1/crates/rand/0.7.2/download
 cargo_pattern = r"^https?://crates.io/api/v1/crates/(?P<name>.+)/(?P<version>.+)(\/download)$"
 
@@ -667,74 +598,6 @@ def build_bitbucket_purl(url):
     )
 
 
-@purl_router.route("https?://gitlab\\.com/(?!.*/archive/).*")
-def build_gitlab_purl(url):
-    """
-    Return a PackageURL object from Gitlab `url`.
-    For example:
-    https://gitlab.com/TG1999/firebase/-/tree/1a122122/views
-    https://gitlab.com/TG1999/firebase/-/tree
-    https://gitlab.com/TG1999/firebase/-/master
-    https://gitlab.com/tg1999/Firebase/-/tree/master
-    https://gitlab.com/tg1999/Firebase/-/commit/bf04e5f289885cf2f20a92b387bcc6df33e30809
-    """
-    # https://gitlab.com/<ns>/<name>/-/commit/<sha>
-    commit_pattern = (
-        r"https?://gitlab.com/"
-        r"(?P<namespace>[^/]+)/(?P<name>[^/]+)/-/commit/"
-        r"(?P<version>[0-9a-fA-F]{7,64})/?$"
-    )
-
-    commit_matche = re.search(commit_pattern, url)
-    if commit_matche:
-        return PackageURL(
-            type="gitlab",
-            namespace=commit_matche.group("namespace"),
-            name=commit_matche.group("name"),
-            version=commit_matche.group("version"),
-            qualifiers={},
-            subpath="",
-        )
-
-    segments = get_path_segments(url)
-
-    if not len(segments) >= 2:
-        return
-    namespace = segments[0]
-    name = segments[1]
-    version = None
-    subpath = None
-
-    # https://gitlab.com/TG1999/firebase/master
-    if (len(segments) >= 3) and segments[2] != "-" and segments[2] != "tree":
-        version = segments[2]
-        subpath = "/".join(segments[3:])
-
-    # https://gitlab.com/TG1999/firebase/-/tree/master
-    if len(segments) >= 5 and (segments[2] == "-" and segments[3] == "tree"):
-        version = segments[4]
-        subpath = "/".join(segments[5:])
-
-    return PackageURL(
-        type="gitlab",
-        namespace=namespace,
-        name=name,
-        version=version,
-        subpath=subpath,
-    )
-
-
-# https://gitlab.com/hoppr/hoppr/-/archive/v1.11.1-dev.2/hoppr-v1.11.1-dev.2.tar.gz
-gitlab_archive_pattern = (
-    r"^https?://gitlab.com/"
-    r"(?P<namespace>.+)/(?P<name>.+)/-/archive/(?P<version>.+)/"
-    r"(?P=name)-(?P=version).*"
-    r"[^/]$"
-)
-
-register_pattern("gitlab", gitlab_archive_pattern)
-
-
 # https://hackage.haskell.org/package/cli-extras-0.2.0.0/cli-extras-0.2.0.0.tar.gz
 hackage_download_pattern = (
     r"^https?://hackage.haskell.org/package/"

tests/contrib/data/url2purl.json

@@ -155,31 +155,6 @@
   "https://www.nuget.org/api/v2/package/MvvmLightLibs/4.1.23": "pkg:nuget/MvvmLightLibs@4.1.23",
   "https://www.nuget.org/api/v2/package/Twilio/3.4.1": "pkg:nuget/Twilio@3.4.1",
   "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/10.0.1/newtonsoft.json.10.0.1.nupkg": "pkg:nuget/newtonsoft.json@10.0.1",
-  "http://master.dl.sourceforge.net/project/zznotes/zznotes/1.1.2/zznotes-1.1.2.tar.gz": "pkg:sourceforge/zznotes/zznotes@1.1.2",
-  "http://master.dl.sourceforge.net/project/zapping/zvbi/0.2.35/zvbi-0.2.35.tar.bz2": "pkg:sourceforge/zapping/zvbi@0.2.35",
-  "http://master.dl.sourceforge.net/project/libpng/zlib/1.2.3/zlib-1.2.3.tar.bz2": "pkg:sourceforge/libpng/zlib@1.2.3",
-  "http://master.dl.sourceforge.net/project/xmlstar/xmlstarlet/1.0.0/xmlstarlet-1.0.0-1.src.rpm": "pkg:sourceforge/xmlstar/xmlstarlet@1.0.0",
-  "http://master.dl.sourceforge.net/project/wxmozilla/wxMozilla/0.5.5/wxMozilla-0.5.5.exe": "pkg:sourceforge/wxmozilla/wxMozilla@0.5.5",
-  "http://iweb.dl.sourceforge.net/project/sblim/sblim-cim-client2/2.2.5/sblim-cim-client2-2.2.5-src.zip": "pkg:sourceforge/sblim/sblim-cim-client2@2.2.5",
-  "http://master.dl.sourceforge.net/project/zinnia/zinnia-win32/0.06/zinnia-win32-0.06.zip": "pkg:sourceforge/zinnia/zinnia-win32@0.06",
-  "http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/": "pkg:sourceforge/findbugs?download_url=http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/",
-  "http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar": "pkg:sourceforge/arestc?download_url=http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar",
-  "http://master.dl.sourceforge.net/project/intraperson/OldFiles/intraperson/0.28/intraperson-0.28.tar.gz": "pkg:sourceforge/intraperson/intraperson@0.28",
-  "http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip": "pkg:sourceforge/pwiki?download_url=http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip",
-  "http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz": "pkg:sourceforge/iswraid?download_url=http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz",
-  "http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%20stable/0.1a1_stable_AloysCore.zip": "pkg:sourceforge/aloyscore?download_url=http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip",
-  "http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip": "pkg:sourceforge/myenterprise?download_url=http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip",
-  "http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip": "pkg:sourceforge/wxhaskell?download_url=http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip",
-  "http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip": "pkg:sourceforge/a2freedom?download_url=http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip",
-  "http://master.dl.sourceforge.net/project/tinyos/OldFiles/tinyos/1.1.0/tinyos-1.1.0.tar.gz": "pkg:sourceforge/tinyos/tinyos@1.1.0",
-  "http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar": "pkg:sourceforge/urlchecker?download_url=http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar",
-  "http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar": "pkg:sourceforge/zclasspath?download_url=http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar",
-  "http://master.dl.sourceforge.net/project/googleimagedown/project/v1.1/GoogleImageDownloader-v1.1-src.tar.bz2": "pkg:sourceforge/googleimagedown?download_url=http://master.dl.sourceforge.net/project/googleimagedown/project/v1.1/GoogleImageDownloader-v1.1-src.tar.bz2",
-  "https://sourceforge.net/projects/scribus/files/scribus/1.6.0/scribus-1.6.0.tar.gz/download": "pkg:sourceforge/scribus@1.6.0",
-  "https://sourceforge.net/projects/turbovnc/files/3.1/turbovnc-3.1.tar.gz/download": "pkg:sourceforge/turbovnc@3.1",
-  "https://sourceforge.net/projects/ventoy/files/v1.0.96/Ventoy%201.0.96%20release%20source%20code.tar.gz/download": "pkg:sourceforge/ventoy@1.0.96",
-  "https://sourceforge.net/projects/geoserver/files/GeoServer/2.23.4/geoserver-2.23.4-war.zip/download": "pkg:sourceforge/geoserver@2.23.4",
-  "https://sourceforge.net/projects/spacesniffer/files/spacesniffer_1_3_0_2.zip/download": "pkg:sourceforge/spacesniffer@1_3_0_2",
   "https://crates.io/api/v1/crates/rand/0.7.2/download": "pkg:cargo/rand@0.7.2",
   "https://crates.io/api/v1/crates/clap/2.33.0/download": "pkg:cargo/clap@2.33.0",
   "https://crates.io/api/v1/crates/structopt/0.3.11/download": "pkg:cargo/structopt@0.3.11",
@@ -253,17 +228,6 @@
   "https://bitbucket.org/multicoreware/x265/downloads/x265_2.6.tar.gz": "pkg:bitbucket/multicoreware/x265?download_url=https://bitbucket.org/multicoreware/x265/downloads/x265_2.6.tar.gz",
   "https://bitbucket.org/robeden/trove/downloads/trove-3.0.3.zip": "pkg:bitbucket/robeden/trove?download_url=https://bitbucket.org/robeden/trove/downloads/trove-3.0.3.zip",
   "https://bitbucket.org/efotinis/deskpins/downloads/DeskPins-1.31-setup.exe": "pkg:bitbucket/efotinis/deskpins?download_url=https://bitbucket.org/efotinis/deskpins/downloads/DeskPins-1.31-setup.exe",
-  "https://gitlab.com/TG1999/firebase/-/tree/1a122122/views": "pkg:gitlab/tg1999/firebase@1a122122#views",
-  "https://gitlab.com/tg1999/firebase": "pkg:gitlab/tg1999/firebase",
-  "https://gitlab.com/TG1999/firebase/-/": "pkg:gitlab/tg1999/firebase",
-  "https://gitlab.com/TG1999/firebase/-/tree": "pkg:gitlab/tg1999/firebase",
-  "https://gitlab.com/TG1999/firebase/-/master": "pkg:gitlab/tg1999/firebase",
-  "https://gitlab.com/TG1999/firebase/tree/": "pkg:gitlab/tg1999/firebase",
-  "https://gitlab.com/TG1999/firebase/master": "pkg:gitlab/tg1999/firebase@master",
-  "https://gitlab.com/TG1999/firebase/-/tree/master": "pkg:gitlab/tg1999/firebase@master",
-  "https://gitlab.com/tg1999/Firebase/-/tree/master": "pkg:gitlab/tg1999/firebase@master",
-  "https://gitlab.com/TG1999/FIREBASE": "pkg:gitlab/tg1999/firebase",
-  "https://gitlab.com/hoppr/hoppr/-/archive/v1.11.1-dev.2/hoppr-v1.11.1-dev.2.tar.gz": "pkg:gitlab/hoppr/hoppr@v1.11.1-dev.2",
   "https://hackage.haskell.org/package/a50-0.5/a50-0.5.tar.gz": "pkg:hackage/a50@0.5",
   "https://hackage.haskell.org/package/AC-HalfInteger-1.2.1/AC-HalfInteger-1.2.1.tar.gz": "pkg:hackage/AC-HalfInteger@1.2.1",
   "https://hackage.haskell.org/package/3d-graphics-examples-0.0.0.2/3d-graphics-examples-0.0.0.2.tar.gz": "pkg:hackage/3d-graphics-examples@0.0.0.2",
@@ -276,6 +240,5 @@
   "https://cran.r-project.org/src/contrib/jsonlite_1.8.8.tar.gz": "pkg:cran/jsonlite@1.8.8",
   "https://packagemanager.rstudio.com/cran/2022-06-23/src/contrib/curl_4.3.2.tar.gz": "pkg:cran/curl@4.3.2?download_url=https://packagemanager.rstudio.com/cran/2022-06-23/src/contrib/curl_4.3.2.tar.gz",
   "https://github.com/TG1999/first_repo/commit/98e516011d6e096e25247b82fc5f196bbeecff10": "pkg:github/tg1999/first_repo@98e516011d6e096e25247b82fc5f196bbeecff10",
-  "https://gitlab.com/TG1999/first_repo/-/commit/bf04e5f289885cf2f20a92b387bcc6df33e30809": "pkg:gitlab/tg1999/first_repo@bf04e5f289885cf2f20a92b387bcc6df33e30809",
   "https://bitbucket.org/TG1999/first_repo/commits/16a60c4a74ef477cd8c16ca82442eaab2fbe8c86": "pkg:bitbucket/tg1999/first_repo@16a60c4a74ef477cd8c16ca82442eaab2fbe8c86"
 }