ENT-12098: Added an initial implementation of an internal cfbs command to generate MPF release information

README.md

@@ -287,6 +287,7 @@ These commands are intended to be run as part of build systems / deployment pipe
   `cfbs set-input` and `cfbs get-input` can be thought of as ways to save and load the input file.
   Similar to `cfbs get-input` the JSON contains both the specification (what the module accepts and how it's presented to the user) as well as the user's responses (if present).
   Expected usage is to run `cfbs get-input` to get the JSON, and then fill out the response part and run `cfbs set-input`.
+* `cfbs generate-release-information`: An internal command used to generate JSON release information files from the [official CFEngine masterfiles](https://github.com/cfengine/masterfiles/).
 * `cfbs validate`: Used to validate the [index JSON file](https://github.com/cfengine/build-index/blob/master/cfbs.json).
   May be expanded to validate other files and formats in the future.
   **Note:** If you use `cfbs validate` as part of your automation, scripts, and build systems, be aware that we might add more strict validation rules in the future, so be prepared to sometimes have it fail after upgrading the version of cfbs.

cfbs/args.py

@@ -100,6 +100,11 @@ def get_arg_parser():
         help="Ignore versions.json. Necessary in case of a custom index or testing changes to the default index.",
         action="store_true",
     )
+    parser.add_argument(
+        "--omit-download",
+        help="Use existing masterfiles instead of downloading in 'cfbs generate-release-information'",
+        action="store_true",
+    )
     parser.add_argument(
         "--masterfiles", help="Add masterfiles on cfbs init choose between"
     )

cfbs/cfbs.1

@@ -1,4 +1,4 @@
-.TH CFBS "1" "2024\-06\-07" "cfbs" "CFEngine Build System manual"
+.TH CFBS "1" "2024\-11\-22" "cfbs" "CFEngine Build System manual"
 .SH NAME
 cfbs \- combines multiple modules into 1 policy set to deploy on your infrastructure. Modules can be custom promise types, JSON files which enable certain functionality, or reusable CFEngine policy. The modules you use can be written by the CFEngine team, others in the community, your colleagues, or yourself.
 .SH SYNOPSIS
@@ -9,7 +9,7 @@ CFEngine Build System.
 
 .TP
 \fBcmd\fR
-The command to perform (pretty, init, status, search, add, remove, clean, update, validate, download, build, install, help, info, show, input, set\-input, get\-input)
+The command to perform (pretty, init, status, search, add, remove, clean, update, validate, download, build, install, help, info, show, input, set\-input, get\-input, generate\-release\-information)
 
 .TP
 \fBargs\fR
@@ -72,6 +72,10 @@ Specify git commit message
 \fB\-\-ignore\-versions\-json\fR
 Ignore versions.json. Necessary in case of a custom index or testing changes to the default index.
 
+.TP
+\fB\-\-omit\-download\fR
+Use existing masterfiles instead of downloading in 'cfbs generate-release-information'
+
 .TP
 \fB\-\-masterfiles\fR \fI\,MASTERFILES\/\fR
 Add masterfiles on cfbs init choose between

cfbs/commands.py

@@ -65,6 +65,7 @@
 from cfbs.git_magic import Result, commit_after_command, git_commit_maybe_prompt
 from cfbs.prompts import YES_NO_CHOICES, prompt_user
 from cfbs.module import Module, is_module_added_manually
+from cfbs.masterfiles.generate_release_information import generate_release_information
 
 
 class InputDataUpdateFailed(Exception):
@@ -1204,3 +1205,8 @@ def get_input_command(name, outfile):
         log.error("Failed to write json: %s" % e)
         return 1
     return 0
+
+
+@cfbs_command("generate-release-information")
+def generate_release_information_command(omit_download=False):
+    generate_release_information(omit_download)

cfbs/main.py

@@ -58,6 +58,12 @@ def main() -> int:
             % args.command
         )
 
+    if args.omit_download and args.command != "generate-release-information":
+        user_error(
+            "The option --omit-download is only for 'cfbs generate-release-information', not 'cfbs %s'"
+            % args.command
+        )
+
     if args.non_interactive and args.command not in (
         "init",
         "add",
@@ -91,6 +97,11 @@ def main() -> int:
     if args.command in ("info", "show"):
         return commands.info_command(args.args)
 
+    if args.command == "generate-release-information":
+        return commands.generate_release_information_command(
+            omit_download=args.omit_download
+        )
+
     if not is_cfbs_repo():
         user_error("This is not a cfbs repo, to get started, type: cfbs init")
 

cfbs/masterfiles/analyze.py

@@ -0,0 +1,127 @@
+from collections import OrderedDict
+import os
+
+from cfbs.utils import dict_sorted_by_key, file_sha256
+
+
+def initialize_vcf():
+    versions_dict = {"versions": {}}
+    checksums_dict = {"checksums": {}}
+    files_dict = {"files": {}}
+
+    return versions_dict, checksums_dict, files_dict
+
+
+def versions_checksums_files(
+    files_dir_path, version, versions_dict, checksums_dict, files_dict
+):
+    for root, _, files in os.walk(files_dir_path):
+        for name in files:
+            full_relpath = os.path.join(root, name)
+            tarball_relpath = os.path.relpath(full_relpath, files_dir_path)
+            file_checksum = file_sha256(full_relpath)
+
+            if version not in versions_dict["versions"]:
+                versions_dict["versions"][version] = {}
+            if "files" not in versions_dict["versions"][version]:
+                versions_dict["versions"][version]["files"] = {}
+            versions_dict["versions"][version]["files"][tarball_relpath] = file_checksum
+
+            if not file_checksum in checksums_dict["checksums"]:
+                checksums_dict["checksums"][file_checksum] = []
+            checksums_dict["checksums"][file_checksum].append(
+                {
+                    "file": tarball_relpath,
+                    "version": version,
+                }
+            )
+
+            if not tarball_relpath in files_dict["files"]:
+                files_dict["files"][tarball_relpath] = []
+            files_dict["files"][tarball_relpath].append(
+                {
+                    "checksum": file_checksum,
+                    "version": version,
+                }
+            )
+
+    return versions_dict, checksums_dict, files_dict
+
+
+def finalize_vcf(versions_dict, checksums_dict, files_dict):
+    # explicitly sort VCF data to ensure determinism
+
+    # checksums.json:
+    working_dict = checksums_dict["checksums"]
+    # sort each list, first by version descending, then by filepath alphabetically
+    for k in working_dict.keys():
+        working_dict[k] = sorted(
+            working_dict[k],
+            key=lambda d: (
+                version_as_comparable_list_negated(d["version"]),
+                d["file"],
+            ),
+        )
+    # sort checksums
+    checksums_dict["checksums"] = dict_sorted_by_key(working_dict)
+
+    # files.json:
+    working_dict = files_dict["files"]
+    # sort each list, first by version descending, then by checksum
+    for k in working_dict.keys():
+        working_dict[k] = sorted(
+            working_dict[k],
+            key=lambda d: (
+                version_as_comparable_list_negated(d["version"]),
+                d["checksum"],
+            ),
+        )
+    # sort files, alphabetically
+    files_dict["files"] = dict_sorted_by_key(working_dict)
+
+    # versions.json:
+    working_dict = versions_dict["versions"]
+    # sort files of each version
+    for k in working_dict.keys():
+        working_dict[k]["files"] = dict_sorted_by_key(working_dict[k]["files"])
+    # sort version numbers, in decreasing order
+    versions_dict["versions"] = OrderedDict(
+        sorted(
+            versions_dict["versions"].items(),
+            key=lambda p: (version_as_comparable_list(p[0]), p[1]),
+            reverse=True,
+        )
+    )
+
+    return versions_dict, checksums_dict, files_dict
+
+
+def version_as_comparable_list(version: str):
+    """Also supports versions containing exactly one of `b` or `-`.
+
+    Example of the version ordering: `3.24.0b1 < 3.24.0 < 3.24.0-1`.
+
+    Examples:
+    * `version_as_comparable_list("3.24.0b1")` is `[[3, 24, 0], [-1, 1]]`
+    * `version_as_comparable_list("3.24.0-2")` is `[[3, 24, 0], [1, 2]]`
+    * `version_as_comparable_list("3.24.x")` is `[[3, 24, 99999], [0, 0]]`"""
+    if "b" not in version:
+        if "-" not in version:
+            version += "|0.0"
+    version = version.replace("x", "99999").replace("-", "|1.").replace("b", "|-1.")
+    versionpair = version.split("|")
+    versionlist = [versionpair[0].split("."), versionpair[1].split(".")]
+
+    versionlist[0] = [int(s) for s in versionlist[0]]
+    versionlist[1] = [int(s) for s in versionlist[1]]
+
+    return versionlist
+
+
+def version_as_comparable_list_negated(version):
+    vcl = version_as_comparable_list(version)
+
+    vcl[0] = [-x for x in vcl[0]]
+    vcl[1] = [-x for x in vcl[1]]
+
+    return vcl

cfbs/masterfiles/check_download_matches_git.py

@@ -0,0 +1,46 @@
+import os
+
+from cfbs.utils import dict_diff, read_json, user_error
+
+
+def check_download_matches_git(versions):
+    """Check that the downloadable files match the git files.
+
+    This can be used to monitor / detect if something has been changed, accidentally or maliciously.
+
+    Generates a `differences-*.txt` file for each version.
+    """
+
+    download_versions_dict = read_json("versions.json")
+    git_versions_dict = read_json("versions-git.json")
+
+    os.makedirs("differences", exist_ok=True)
+
+    for version in versions:
+        download_version_dict = download_versions_dict["versions"][version]["files"]
+        git_version_dict = git_versions_dict["versions"][version]["files"]
+
+        # normalize downloaded version dictionary filepaths
+        # necessary because the downloaded version and git version dictionaries have filepaths of different forms
+        new_download_dict = {}
+        for key, value in download_version_dict.items():
+            if key.startswith("masterfiles/"):
+                key = key[12:]
+            new_download_dict[key] = value
+        download_version_dict = new_download_dict
+
+        with open("differences/difference-" + version + ".txt", "w") as f:
+            only_dl, only_git, value_diff = dict_diff(
+                download_version_dict, git_version_dict
+            )
+
+            print("Files only in the downloaded version:", only_dl, file=f)
+            print("Files only in the git version:", only_git, file=f)
+            print("Files with different contents:", value_diff, file=f)
+
+            if len(only_dl) > 0 or len(value_diff) > 0:
+                user_error(
+                    "Downloadable files of version "
+                    + version
+                    + " do not match git files"
+                )

cfbs/masterfiles/download_all_versions.py

@@ -0,0 +1,115 @@
+import os
+import shutil
+
+from cfbs.utils import FetchError, fetch_url, get_json, mkdir, user_error
+
+ENTERPRISE_RELEASES_URL = "https://cfengine.com/release-data/enterprise/releases.json"
+
+
+def get_download_urls_enterprise():
+    download_urls = {}
+    reported_checksums = {}
+
+    print("* gathering download URLs...")
+
+    data = get_json(ENTERPRISE_RELEASES_URL)
+
+    for release_data in data["releases"]:
+        version = release_data["version"]
+
+        if version == "3.10.0":
+            # for 3.10.0, for some reason, the "Masterfiles ready-to-install tarball" is a .tar.gz tarball, rather than a .pkg.tar.gz tarball
+            # download the .pkg.tar.gz tarball from an unlisted analoguous URL instead
+            download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.10.0.pkg.tar.gz"
+            digest = "7b5e237529e11ce4ae295922dad1a681f13b95f3a7d247d39d3f5088f1a1d7d3"
+            download_urls[version] = download_url
+            reported_checksums[version] = digest
+            continue
+        if version == "3.9.2":
+            # for 3.9.2, no masterfiles are listed, but an unlisted analoguous URL exists
+            download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.9.2.pkg.tar.gz"
+            digest = "ae1a758530d4a4aad5b6812b61fc37ad1b5900b755f88a1ab98da7fd05a9f5cc"
+            download_urls[version] = download_url
+            reported_checksums[version] = digest
+            continue
+
+        release_url = release_data["URL"]
+        subdata = get_json(release_url)
+        artifacts_data = subdata["artifacts"]
+
+        if "Additional Assets" not in artifacts_data:
+            # happens for 3.9.0b1, 3.8.0b1, 3.6.1, 3.6.0
+            continue
+
+        assets_data = artifacts_data["Additional Assets"]
+        masterfiles_data = None
+
+        for asset in assets_data:
+            if asset["Title"] == "Masterfiles ready-to-install tarball":
+                masterfiles_data = asset
+
+        if masterfiles_data is None:
+            # happens for 3.9.2, 3.9.0, 3.8.2, 3.8.1, 3.8.0, 3.7.4--3.6.2
+            # 3.9.2: see above
+            # 3.9.0 and below: no masterfiles listed, and unlisted analogous URLs seemingly do not exist
+            continue
+
+        download_urls[version] = masterfiles_data["URL"]
+        reported_checksums[version] = masterfiles_data["SHA256"]
+
+    return download_urls, reported_checksums
+
+
+def download_versions_from_urls(download_path, download_urls, reported_checksums):
+    downloaded_versions = []
+
+    mkdir(download_path)
+
+    for version, url in download_urls.items():
+        # ignore master and .x versions
+        if url.startswith("http://buildcache"):
+            continue
+
+        print("* downloading from", url)
+        downloaded_versions.append(version)
+
+        version_path = os.path.join(download_path, version)
+        mkdir(version_path)
+
+        # download a version, and verify the reported checksum matches
+        filename = url.split("/")[-1]
+        tarball_path = os.path.join(version_path, filename)
+        checksum = reported_checksums[version]
+        try:
+            fetch_url(url, tarball_path, checksum)
+        except FetchError as e:
+            user_error("For version " + version + ": " + str(e))
+
+        tarball_dir_path = os.path.join(version_path, "tarball")
+        shutil.unpack_archive(tarball_path, tarball_dir_path)
+
+    return downloaded_versions
+
+
+def download_all_versions(download_path):
+    download_urls, reported_checksums = get_download_urls_enterprise()
+
+    # add masterfiles versions which do not appear in Enterprise releases but appear in Community releases
+    # 3.12.0b1
+    version = "3.12.0b1"
+    download_url = "https://cfengine-package-repos.s3.amazonaws.com/community_binaries/Community-3.12.0b1/misc/cfengine-masterfiles-3.12.0b1.pkg.tar.gz"
+    digest = "ede305dae7be3edfac04fc5b7f63b46adb3a5b1612f4755e855ee8e6b8d344d7"
+    download_urls[version] = download_url
+    reported_checksums[version] = digest
+    # 3.10.0b1
+    version = "3.10.0b1"
+    download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.10.0b1.pkg.tar.gz"
+    digest = "09291617254705d79dea2531b23dbd0754f09029e90ce0b43b275aa02c1223a3"
+    download_urls[version] = download_url
+    reported_checksums[version] = digest
+
+    downloaded_versions = download_versions_from_urls(
+        download_path, download_urls, reported_checksums
+    )
+
+    return downloaded_versions