Add workaround to accept GL canonical names without a commit ID

Fixes: #302
Signed-off-by: Tobias Wolf <wolf@b1-systems.de>
On-behalf-of: SAP <tobias.wolf@sap.com>

Author: NotTheEvilOne

.github/actions/features_parse/action.yml

@@ -11,7 +11,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.10.10
+    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.10.11
     - id: result
       shell: bash
       run: |

.github/actions/flavors_parse/action.yml

@@ -13,7 +13,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.10.10
+    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.10.11
     - id: matrix
       shell: bash
       run: |

.github/actions/setup/action.yml

@@ -4,7 +4,7 @@ description: Installs the given GardenLinux Python library
 inputs:
   version:
     description: GardenLinux Python library version
-    default: "0.10.10"
+    default: "0.10.11"
   python_version:
     description: Python version to setup
     default: "3.13"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gardenlinux"
-version = "0.10.10"
+version = "0.10.11"
 description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames"
 authors = ["Garden Linux Maintainers <contact@gardenlinux.io>"]
 license = "Apache-2.0"

src/gardenlinux/features/cname.py

@@ -69,17 +69,46 @@ def __init__(
 
         commit_id_or_hash = None
 
-        re_match = re.match(
-            "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)*?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$",
-            cname,
+        if version is not None:
+            # Support version values formatted as <version>-<commit_hash>
+            if commit_hash is None:
+                re_match = re.match("([a-z0-9.]+)(-([a-z0-9]+))?$", version)
+                assert re_match, f"Not a valid version {version}"
+
+                commit_id_or_hash = re_match[3]
+                version = re_match[1]
+            else:
+                commit_id_or_hash = commit_hash
+
+        re_object = re.compile(
+            "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)*?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$"
         )
 
+        re_match = re_object.match(cname)
+
+        # Workaround Garden Linux canonical names without mandatory final commit hash
+        if (
+            not re_match
+            and commit_id_or_hash
+            and re.match(
+                "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)*?)(-([a-z0-9]+)(-([a-z0-9.]+))*)?$",
+                cname,
+            )
+        ):
+            re_match = re_object.match(f"{cname}-{commit_id_or_hash}")
+
         assert re_match, f"Not a valid Garden Linux canonical name {cname}"
 
         if re_match.lastindex == 1:
             self._flavor = re_match[1]
         else:
-            commit_id_or_hash = re_match[7]
+            if commit_id_or_hash is None:
+                commit_id_or_hash = re_match[7]
+            elif re_match.group(7) is not None:
+                assert commit_id_or_hash.startswith(re_match[7]), (
+                    f"Mismatch between Garden Linux canonical name '{cname}' and given commit ID '{commit_id_or_hash}' detected"
+                )
+
             self._flavor = re_match[1]
             self._version = re_match[6]
 
@@ -91,17 +120,13 @@ def __init__(
         if self._arch is None and arch is not None:
             self._arch = arch
 
-        if self._version is None and version is not None:
-            # Support version values formatted as <version>-<commit_hash>
-            if commit_hash is None:
-                re_match = re.match("([a-z0-9.]+)(-([a-z0-9]+))?$", version)
-                assert re_match, f"Not a valid version {version}"
-
-                commit_id_or_hash = re_match[3]
-                self._version = re_match[1]
-            else:
-                commit_id_or_hash = commit_hash
+        if version is not None:
+            if self._version is None:
                 self._version = version
+            else:
+                assert version == self._version, (
+                    f"Mismatch between Garden Linux canonical name '{cname}' and given version '{version}' detected"
+                )
 
         if commit_id_or_hash is not None:
             self._commit_id = commit_id_or_hash[:8]
@@ -312,21 +337,36 @@ def release_metadata_string(self) -> str:
             assert len(features["platform"]) < 2
             "Only one platform is supported"
 
+        commit_hash = self.commit_hash
+        commit_id = self.commit_id
         elements = ",".join(features["element"])
         flags = ",".join(features["flag"])
         platform = features["platform"][0]
         platforms = ",".join(features["platform"])
         platform_variant = self.platform_variant
+        version = self.version
+
+        if commit_id is None:
+            commit_id = ""
+
+        if commit_hash is None:
+            commit_hash = commit_id
 
         if platform_variant is None:
             platform_variant = ""
 
+        if version is None:
+            pretty_name = f"{GL_DISTRIBUTION_NAME} unsupported version"
+            version = ""
+        else:
+            pretty_name = f"{GL_DISTRIBUTION_NAME} {version}"
+
         metadata = f"""
 ID={GL_RELEASE_ID}
 ID_LIKE=debian
 NAME="{GL_DISTRIBUTION_NAME}"
-PRETTY_NAME="{GL_DISTRIBUTION_NAME} {self.version}"
-IMAGE_VERSION={self.version}
+PRETTY_NAME="{pretty_name}"
+IMAGE_VERSION={version}
 VARIANT_ID="{self.flavor}-{self.arch}"
 HOME_URL="{GL_HOME_URL}"
 SUPPORT_URL="{GL_SUPPORT_URL}"
@@ -338,9 +378,9 @@ def release_metadata_string(self) -> str:
 GARDENLINUX_FEATURES_FLAGS="{flags}"
 GARDENLINUX_PLATFORM="{platform}"
 GARDENLINUX_PLATFORM_VARIANT="{platform_variant}"
-GARDENLINUX_VERSION="{self.version}"
-GARDENLINUX_COMMIT_ID="{self.commit_id}"
-GARDENLINUX_COMMIT_ID_LONG="{self.commit_hash}"
+GARDENLINUX_VERSION="{version}"
+GARDENLINUX_COMMIT_ID="{commit_id}"
+GARDENLINUX_COMMIT_ID_LONG="{commit_hash}"
         """.strip()
 
         return metadata
@@ -365,7 +405,7 @@ def version_and_commit_id(self) -> Optional[str]:
         :since:  0.7.0
         """
 
-        if self._commit_id is None:
+        if self._version is None or self._commit_id is None:
             return None
 
         return f"{self._version}-{self._commit_id}"

src/gardenlinux/s3/__main__.py

@@ -9,11 +9,6 @@
 
 from .s3_artifacts import S3Artifacts
 
-_ARGS_ACTION_ALLOWED = [
-    "download-artifacts-from-bucket",
-    "upload-artifacts-to-bucket",
-]
-
 
 def main() -> None:
     """
@@ -25,17 +20,22 @@ def main() -> None:
     parser = argparse.ArgumentParser()
 
     parser.add_argument("--bucket", dest="bucket")
-    parser.add_argument("--cname", required=False, dest="cname")
     parser.add_argument("--path", required=False, dest="path")
     parser.add_argument("--dry-run", action="store_true")
 
-    parser.add_argument("action", nargs="?", choices=_ARGS_ACTION_ALLOWED)
+    subparsers = parser.add_subparsers(dest="action")
+
+    download_parser = subparsers.add_parser("download-artifacts-from-bucket")
+    download_parser.add_argument("--cname", required=False, dest="cname")
+
+    upload_parser = subparsers.add_parser("upload-artifacts-to-bucket")
+    upload_parser.add_argument("--artifact-name", required=False, dest="artifact_name")
 
     args = parser.parse_args()
 
     if args.action == "download-artifacts-from-bucket":
         S3Artifacts(args.bucket).download_to_directory(args.cname, args.path)
     elif args.action == "upload-artifacts-to-bucket":
         S3Artifacts(args.bucket).upload_from_directory(
-            args.cname, args.path, dry_run=args.dry_run
+            args.artifact_name, args.path, dry_run=args.dry_run
         )

src/gardenlinux/s3/s3_artifacts.py

@@ -98,15 +98,15 @@ def download_to_directory(
 
     def upload_from_directory(
         self,
-        cname: str,
+        artifact_base_name: str,
         artifacts_dir: PathLike[str] | str,
         delete_before_push: bool = False,
         dry_run: bool = False,
     ) -> None:
         """
         Pushes S3 artifacts to the underlying bucket.
 
-        :param cname:              Canonical name of the GardenLinux S3 artifacts
+        :param artifact_base_name: Base name of the GardenLinux S3 artifacts
         :param artifacts_dir:      Path of the image artifacts
         :param delete_before_push: True to delete objects before upload
 
@@ -115,35 +115,32 @@ def upload_from_directory(
 
         artifacts_dir = Path(artifacts_dir)
 
-        cname_object = CName(cname)
-
         if not artifacts_dir.is_dir():
             raise RuntimeError(f"Artifacts directory given is invalid: {artifacts_dir}")
 
-        release_file = artifacts_dir.joinpath(f"{cname}.release")
-        release_timestamp = stat(release_file).st_ctime
-
-        cname_object.load_from_release_file(release_file)
+        release_file = artifacts_dir.joinpath(f"{artifact_base_name}.release")
 
-        if cname_object.arch is None:
-            raise RuntimeError(
-                "Architecture could not be determined from GardenLinux canonical name or release file"
-            )
+        cname_object = CName.new_from_release_file(release_file)
 
         if cname_object.version_and_commit_id is None:
             raise RuntimeError(
-                "Version information could not be determined from GardenLinux canonical name or release file"
+                "Version information could not be determined from release file"
             )
 
+        arch = cname_object.arch
         feature_list = cname_object.feature_set
-        requirements_file = artifacts_dir.joinpath(f"{cname}.requirements")
+        release_timestamp = stat(release_file).st_ctime
+        requirements_file = artifacts_dir.joinpath(f"{artifact_base_name}.requirements")
         require_uefi = None
         secureboot = None
 
         if requirements_file.exists():
             requirements_config = ConfigParser(allow_unnamed_section=True)
             requirements_config.read(requirements_file)
 
+            if requirements_config.has_option(UNNAMED_SECTION, "arch"):
+                arch = requirements_config.get(UNNAMED_SECTION, "arch")
+
             if requirements_config.has_option(UNNAMED_SECTION, "uefi"):
                 require_uefi = requirements_config.getboolean(UNNAMED_SECTION, "uefi")
 
@@ -152,16 +149,25 @@ def upload_from_directory(
                     UNNAMED_SECTION, "secureboot"
                 )
 
+        if arch is None:
+            raise RuntimeError(
+                "Architecture could not be determined from release or requirements file"
+            )
+
         if require_uefi is None:
             require_uefi = "_usi" in feature_list
 
         if secureboot is None:
             secureboot = "_trustedboot" in feature_list
 
-        commit_hash = cname_object.commit_hash
+        # RegEx for S3 supported characters
+        re_object = re.compile("[^a-zA-Z0-9\\s+\\-=.\\_:/@]")
+
+        arch = re_object.sub("+", arch)
+        commit_id_or_hash = cname_object.commit_hash
 
-        if commit_hash is None:
-            commit_hash = ""
+        if commit_id_or_hash is None:
+            commit_id_or_hash = cname_object.commit_id
 
         version_epoch = str(cname_object.version_epoch)
 
@@ -170,9 +176,9 @@ def upload_from_directory(
 
         metadata = {
             "platform": cname_object.feature_set_platform,
-            "architecture": cname_object.arch,
+            "architecture": arch,
             "base_image": None,
-            "build_committish": commit_hash,
+            "build_committish": commit_id_or_hash,
             "build_timestamp": datetime.fromtimestamp(release_timestamp).isoformat(),
             "gardenlinux_epoch": {version_epoch},
             "logs": None,
@@ -181,7 +187,7 @@ def upload_from_directory(
             "secureboot": secureboot,
             "published_image_metadata": None,
             "s3_bucket": self._bucket.name,
-            "s3_key": f"meta/singles/{cname}",
+            "s3_key": f"meta/singles/{artifact_base_name}",
             "test_result": None,
             "version": cname_object.version,
             "paths": [],
@@ -192,24 +198,17 @@ def upload_from_directory(
         if platform_variant is not None:
             metadata["platform_variant"] = platform_variant
 
-        re_object = re.compile("[^a-zA-Z0-9\\s+\\-=.\\_:/@]")
-
         for artifact in artifacts_dir.iterdir():
-            if not artifact.match(f"{cname}*"):
+            if not artifact.match(f"{artifact_base_name}*"):
                 continue
 
-            if not artifact.name.startswith(cname):
-                raise RuntimeError(
-                    f"Artifact name '{artifact.name}' does not start with cname '{cname}'"
-                )
-
-            s3_key = f"objects/{cname}/{artifact.name}"
+            s3_key = f"objects/{artifact_base_name}/{artifact.name}"
 
             with artifact.open("rb") as fp:
                 md5sum = file_digest(fp, "md5").hexdigest()
                 sha256sum = file_digest(fp, "sha256").hexdigest()
 
-            suffix = artifact.name[len(cname) :]
+            suffix = artifact.stem
 
             artifact_metadata = {
                 "name": artifact.name,
@@ -221,10 +220,10 @@ def upload_from_directory(
             }
 
             s3_tags = {
-                "architecture": re_object.sub("+", cname_object.arch),
+                "architecture": arch,
                 "platform": re_object.sub("+", cname_object.platform),
                 "version": re_object.sub("+", cname_object.version),  # type: ignore[arg-type]
-                "committish": commit_hash,
+                "committish": commit_id_or_hash,
                 "md5sum": md5sum,
                 "sha256sum": sha256sum,
             }
@@ -246,13 +245,13 @@ def upload_from_directory(
         else:
             if delete_before_push:
                 self._bucket.delete_objects(
-                    Delete={"Objects": [{"Key": f"meta/singles/{cname}"}]}
+                    Delete={"Objects": [{"Key": f"meta/singles/{artifact_base_name}"}]}
                 )
 
             with TemporaryFile(mode="wb+") as fp:
                 fp.write(yaml.dump(metadata).encode("utf-8"))
                 fp.seek(0)
 
                 self._bucket.upload_fileobj(
-                    fp, f"meta/singles/{cname}", ExtraArgs={"ContentType": "text/yaml"}
+                    fp, f"meta/singles/{artifact_base_name}", ExtraArgs={"ContentType": "text/yaml"}
                 )

tests/s3/conftest.py

@@ -27,12 +27,12 @@ def make_cname(
     flavor: str = "container",
     arch: str = "amd64",
     version: str = "1234.1",
-    commit: str = "abc123",
+    commit: str = "abc123long",
 ) -> str:
     """
     Helper function to build cname. Can be used to customized the cname.
     """
-    return f"{flavor}-{arch}-{version}-{commit}"
+    return f"{flavor}-{arch}-{version}-{commit[:8]}"
 
 
 # Helpers to compute digests for fake files