Bug 1882852 - remove vendored createprecomplete in iscript and signingscript

This needed to be kept in sync with the code in mozilla-central, or bad
things happen, which bit us in
https://bugzilla.mozilla.org/show_bug.cgi?id=1882322

Instead, directly add remove instructions for the extra signature files
we're adding, leaving the rest of the file unchanged.

Author: jcristau

iscript/src/iscript/autograph.py

@@ -15,7 +15,6 @@
 from mozpack import mozjar
 from requests_hawk import HawkAuth
 
-from iscript.createprecomplete import generate_precomplete
 from iscript.exceptions import IScriptError
 from scriptworker_client.aio import raise_future_exceptions, retry_async
 from scriptworker_client.utils import makedirs, rm
@@ -85,6 +84,7 @@ async def sign_widevine_dir(config, sign_config, app_dir):
     """
     log.info(f"Signing widevine in {app_dir}...")
     all_files = []
+    signed_files = {}
     for top_dir, dirs, files in os.walk(app_dir):
         for file_ in files:
             all_files.append(os.path.join(top_dir, file_))
@@ -98,11 +98,12 @@ async def sign_widevine_dir(config, sign_config, app_dir):
             makedirs(os.path.dirname(to))
             tasks.append(asyncio.ensure_future(sign_widevine_with_autograph(sign_config, from_, "blessed" in fmt, to=to)))
             all_files.append(to)
+            signed_files[from_] = to
         await raise_future_exceptions(tasks)
         remove_extra_files(app_dir, all_files)
-        # Regenerate the `precomplete` file, which is used for cleanup before
+        # Update the `precomplete` file, which is used for cleanup before
         # applying a complete mar.
-        _run_generate_precomplete(config, app_dir)
+        _update_precomplete(config, app_dir, signed_files)
     return app_dir
 
 
@@ -142,16 +143,25 @@ def _get_widevine_signing_files(file_list):
     return files
 
 
-# _run_generate_precomplete {{{1
-def _run_generate_precomplete(config, app_dir):
+# _update_precomplete {{{1
+def _update_precomplete(config, app_dir, signed_files):
     """Regenerate `precomplete` file with widevine sig paths for complete mar."""
     log.info("Generating `precomplete` file...")
-    path = _ensure_one_precomplete(app_dir, "before")
-    with open(path, "r") as fh:
+    precomplete = _ensure_one_precomplete(app_dir)
+    with open(precomplete, "r") as fh:
         before = fh.readlines()
-    generate_precomplete(os.path.dirname(path))
-    path = _ensure_one_precomplete(app_dir, "after")
-    with open(path, "r") as fh:
+    with open(precomplete, "w") as fh:
+        for line in before:
+            fh.write(line)
+            instr, path = line.strip().split(None, 1)
+            if instr != "remove":
+                continue
+            if not path or path[0] != '"' or path[-1] != '"':
+                continue
+            if path[1:-1] not in signed_files:
+                continue
+            fh.write('remove "{}"\n'.format(signed_files[path[1:-1]]))
+    with open(precomplete, "r") as fh:
         after = fh.readlines()
     # Create diff file
     makedirs(os.path.join(config["artifact_dir"], "public", "logs"))
@@ -162,13 +172,13 @@ def _run_generate_precomplete(config, app_dir):
 
 
 # _ensure_one_precomplete {{{1
-def _ensure_one_precomplete(tmp_dir, adj):
+def _ensure_one_precomplete(tmp_dir):
     """Ensure we only have one `precomplete` file in `tmp_dir`."""
     precompletes = glob.glob(os.path.join(tmp_dir, "**", "precomplete"), recursive=True)
     if len(precompletes) < 1:
         raise IScriptError('No `precomplete` file found in "%s"', tmp_dir)
     if len(precompletes) > 1:
-        raise IScriptError('More than one `precomplete` file %s in "%s"', adj, tmp_dir)
+        raise IScriptError('More than one `precomplete` file in "%s"', tmp_dir)
     return precompletes[0]
 
 

iscript/src/iscript/createprecomplete.py

@@ -135,8 +135,7 @@ def fake_isfile(path):
 
     mocker.patch.object(autograph, "sign_widevine_with_autograph", new=noop_async)
     mocker.patch.object(autograph, "makedirs", new=noop_sync)
-    mocker.patch.object(autograph, "generate_precomplete", new=noop_sync)
-    mocker.patch.object(autograph, "_run_generate_precomplete", new=noop_sync)
+    mocker.patch.object(autograph, "_update_precomplete", new=noop_sync)
     mocker.patch.object(os.path, "isfile", new=fake_isfile)
     mocker.patch.object(os, "walk", new=fake_walk)
 
@@ -172,22 +171,21 @@ def test_get_widevine_signing_files(filenames, expected):
     assert autograph._get_widevine_signing_files(filenames) == expected
 
 
-# _run_generate_precomplete {{{1
+# _update_precomplete {{{1
 @pytest.mark.parametrize("num_precomplete,raises", ((1, False), (0, True), (2, True)))
-def test_run_generate_precomplete(tmp_path, num_precomplete, raises, mocker):
-    mocker.patch.object(autograph, "generate_precomplete", new=noop_sync)
+def test_update_precomplete(tmp_path, num_precomplete, raises, mocker):
     work_dir = tmp_path / "work"
     config = {"artifact_dir": tmp_path / "artifacts"}
     for i in range(0, num_precomplete):
         path = os.path.join(work_dir, "foo", str(i))
         makedirs(path)
         with open(os.path.join(path, "precomplete"), "w") as fh:
-            fh.write("blah")
+            fh.write('remove "blah"\n')
     if raises:
         with pytest.raises(IScriptError):
-            autograph._run_generate_precomplete(config, work_dir)
+            autograph._update_precomplete(config, work_dir, {})
     else:
-        autograph._run_generate_precomplete(config, work_dir)
+        autograph._update_precomplete(config, work_dir, {})
 
 
 # remove_extra_files {{{1

iscript/tests/test_autograph.py

@@ -58,5 +58,3 @@ addopts = -vv -s --color=yes
 
 [coverage:run]
 branch = true
-omit =
-    src/iscript/createprecomplete.py

iscript/tox.ini

@@ -30,7 +30,6 @@
 from winsign.crypto import load_pem_certs
 
 from signingscript import task, utils
-from signingscript.createprecomplete import generate_precomplete
 from signingscript.exceptions import SigningScriptError
 from signingscript.rcodesign import RCodesignError, rcodesign_notarize, rcodesign_notary_wait, rcodesign_staple
 
@@ -307,18 +306,20 @@ async def sign_widevine_zip(context, orig_path, fmt):
         # Extract all files so we can create `precomplete` with the full
         # file list
         all_files = await _extract_zipfile(context, orig_path, tmp_dir=tmp_dir)
+        signed_files = {}
         tasks = []
         # Sign the appropriate inner files
         for from_, fmt in files_to_sign.items():
             from_ = os.path.join(tmp_dir, from_)
             to = f"{from_}.sig"
             tasks.append(asyncio.ensure_future(sign_widevine_with_autograph(context, from_, "blessed" in fmt, to=to)))
             all_files.append(to)
+            signed_files[from_] = to
         await raise_future_exceptions(tasks)
         remove_extra_files(tmp_dir, all_files)
-        # Regenerate the `precomplete` file, which is used for cleanup before
+        # Update the `precomplete` file, which is used for cleanup before
         # applying a complete mar.
-        _run_generate_precomplete(context, tmp_dir)
+        _update_precomplete(context, tmp_dir, signed_files)
         await _create_zipfile(context, orig_path, all_files, mode="w", tmp_dir=tmp_dir)
     return orig_path
 
@@ -358,6 +359,7 @@ async def sign_widevine_tar(context, orig_path, fmt):
         # Extract all files so we can create `precomplete` with the full
         # file list
         all_files = await _extract_tarfile(context, orig_path, compression, tmp_dir=tmp_dir)
+        signed_files = {}
         tasks = []
         # Sign the appropriate inner files
         for from_, fmt in files_to_sign.items():
@@ -371,11 +373,12 @@ async def sign_widevine_tar(context, orig_path, fmt):
             makedirs(os.path.dirname(to))
             tasks.append(asyncio.ensure_future(sign_widevine_with_autograph(context, from_, "blessed" in fmt, to=to)))
             all_files.append(to)
+            signed_files[from_] = to
         await raise_future_exceptions(tasks)
         remove_extra_files(tmp_dir, all_files)
-        # Regenerate the `precomplete` file, which is used for cleanup before
+        # Update the `precomplete` file, which is used for cleanup before
         # applying a complete mar.
-        _run_generate_precomplete(context, tmp_dir)
+        _update_precomplete(context, tmp_dir, signed_files)
         await _create_tarfile(context, orig_path, all_files, compression, tmp_dir=tmp_dir)
     return orig_path
 
@@ -577,16 +580,25 @@ def _get_omnija_signing_files(file_list):
     return files
 
 
-# _run_generate_precomplete {{{1
-def _run_generate_precomplete(context, tmp_dir):
+# _update_precomplete {{{1
+def _update_precomplete(context, tmp_dir, signed_files):
     """Regenerate `precomplete` file with widevine sig paths for complete mar."""
     log.info("Generating `precomplete` file...")
-    path = _ensure_one_precomplete(tmp_dir, "before")
-    with open(path, "r") as fh:
+    precomplete = _ensure_one_precomplete(tmp_dir)
+    with open(precomplete, "r") as fh:
         before = fh.readlines()
-    generate_precomplete(os.path.dirname(path))
-    path = _ensure_one_precomplete(tmp_dir, "after")
-    with open(path, "r") as fh:
+    with open(precomplete, "w") as fh:
+        for line in before:
+            fh.write(line)
+            instr, path = line.strip().split(None, 1)
+            if instr != "remove":
+                continue
+            if not path or path[0] != '"' or path[-1] != '"':
+                continue
+            if path[1:-1] not in signed_files:
+                continue
+            fh.write('remove "{}"\n'.format(signed_files[path[1:-1]]))
+    with open(precomplete, "r") as fh:
         after = fh.readlines()
     # Create diff file
     diff_path = os.path.join(context.config["work_dir"], "precomplete.diff")
@@ -597,14 +609,14 @@ def _run_generate_precomplete(context, tmp_dir):
 
 
 # _ensure_one_precomplete {{{1
-def _ensure_one_precomplete(tmp_dir, adj):
+def _ensure_one_precomplete(tmp_dir):
     """Ensure we only have one `precomplete` file in `tmp_dir`."""
     return get_single_item_from_sequence(
         glob.glob(os.path.join(tmp_dir, "**", "precomplete"), recursive=True),
         condition=lambda _: True,
         ErrorClass=SigningScriptError,
         no_item_error_message='No `precomplete` file found in "{}"'.format(tmp_dir),
-        too_many_item_error_message='More than one `precomplete` file {} in "{}"'.format(adj, tmp_dir),
+        too_many_item_error_message='More than one `precomplete` file in "{}"'.format(tmp_dir),
     )