refactor!: remove support for hg sparse-profiles

docs/reference/migrations.rst

@@ -3,6 +3,13 @@ Migration Guide
 
 This page can help when migrating Taskgraph across major versions.
 
+17.x -> 18.x
+------------
+
+* Stop setting the ``run.sparse-profile`` key in all tasks which perform a
+  Mercurial clone. If sparse profiles are still required, the task must perform
+  its own clone and not rely on the ``run-task`` script to do it.
+
 16.x -> 17.x
 ------------
 

src/taskgraph/run-task/run-task

@@ -824,7 +824,6 @@ def hg_checkout(
     head_repo: str,
     base_repo: Optional[str],
     store_path: str,
-    sparse_profile: Optional[str],
     branch: Optional[str],
     revision: Optional[str],
 ):
@@ -849,8 +848,6 @@ def hg_checkout(
 
     if base_repo:
         args.extend(["--upstream", base_repo])
-    if sparse_profile:
-        args.extend(["--sparseprofile", sparse_profile])
 
     # Specify method to checkout a revision. This defaults to revisions as
     # SHA-1 strings, but also supports symbolic revisions like `tip` via the
@@ -914,10 +911,6 @@ def add_vcs_arguments(parser, project, name):
         f"--{project}-checkout",
         help=f"Directory where {name} checkout should be created",
     )
-    parser.add_argument(
-        f"--{project}-sparse-profile",
-        help=f"Path to sparse profile for {name} checkout",
-    )
     parser.add_argument(
         f"--{project}-shallow-clone",
         action="store_true",
@@ -927,7 +920,6 @@ def add_vcs_arguments(parser, project, name):
 
 def collect_vcs_options(args, project, name):
     checkout = getattr(args, f"{project}_checkout")
-    sparse_profile = getattr(args, f"{project}_sparse_profile")
     shallow_clone = getattr(args, f"{project}_shallow_clone")
 
     env_prefix = project.upper()
@@ -964,7 +956,6 @@ def collect_vcs_options(args, project, name):
         "name": name,
         "env-prefix": env_prefix,
         "checkout": checkout,
-        "sparse-profile": sparse_profile,
         "base-repo": base_repo,
         "base-rev": base_rev,
         "head-repo": head_repo,
@@ -1030,7 +1021,6 @@ def vcs_checkout_from_args(options):
                 options["head-repo"],
                 options["base-repo"],
                 options["store-path"],
-                options["sparse-profile"],
                 head_ref,
                 head_rev,
             )

src/taskgraph/transforms/run/common.py

@@ -82,7 +82,7 @@ def generic_worker_add_artifacts(config, task, taskdesc):
     add_artifacts(config, task, taskdesc, path=get_artifact_prefix(taskdesc))
 
 
-def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False):
+def support_vcs_checkout(config, task, taskdesc, repo_configs):
     """Update a task with parameters to enable a VCS checkout.
 
     This can only be used with ``run-task`` tasks, as the cache name is

src/taskgraph/transforms/run/run_task.py

@@ -66,15 +66,6 @@
                 """.lstrip()
             ),
         ): str,
-        Required(
-            "sparse-profile",
-            description=dedent(
-                """
-                The sparse checkout profile to use. Value is the filename relative to the
-                directory where sparse profiles are defined (build/sparse-profiles/).
-                """.lstrip()
-            ),
-        ): Any(str, None),
         Required(
             "command",
             description=dedent(
@@ -140,21 +131,12 @@ def common_setup(config, task, taskdesc, command):
             task,
             taskdesc,
             repo_configs=repo_configs,
-            sparse=bool(run["sparse-profile"]),
         )
 
         for repo_config in repo_configs.values():
             checkout_path = path.join(vcs_path, repo_config.path)
             command.append(f"--{repo_config.prefix}-checkout={checkout_path}")
 
-        if run["sparse-profile"]:
-            command.append(
-                "--{}-sparse-profile=build/sparse-profiles/{}".format(
-                    repo_config.prefix,  # type: ignore
-                    run["sparse-profile"],
-                )
-            )
-
         if "cwd" in run:
             run["cwd"] = path.normpath(run["cwd"].format(checkout=vcs_path))
     elif "cwd" in run and "{checkout}" in run["cwd"]:
@@ -174,7 +156,6 @@ def common_setup(config, task, taskdesc, command):
 
 worker_defaults = {
     "checkout": True,
-    "sparse-profile": None,
     "run-as-root": False,
 }
 

src/taskgraph/transforms/run/toolchain.py

@@ -50,18 +50,6 @@
                 """
             ),
         ): [str],
-        Required(
-            "sparse-profile",
-            description=dedent(
-                """
-                Sparse profile to give to checkout using `run-task`. If given,
-                a filename in `build/sparse-profiles`. Defaults to
-                "toolchain-build", i.e., to
-                `build/sparse-profiles/toolchain-build`. If `None`, instructs
-                `run-task` to not use a sparse profile at all.
-                """
-            ),
-        ): Any(str, None),
         Optional(
             "resources",
             description=dedent(
@@ -205,9 +193,7 @@ def common_toolchain(config, task, taskdesc, is_docker):
     configure_taskdesc_for_run(config, task, taskdesc, worker["implementation"])
 
 
-toolchain_defaults = {
-    "sparse-profile": "toolchain-build",
-}
+toolchain_defaults = {}
 
 
 @run_task_using(

src/taskgraph/util/caches.py

@@ -37,11 +37,6 @@ def get_checkout_cache_name(config: "TransformConfig", task: dict[str, Any]) ->
         digest = hashlib.sha256(checkout_paths_str).hexdigest()
         cache_name += f"-repos-{digest}"
 
-    # Sparse checkouts need their own cache because they can interfere
-    # with clients that aren't sparse aware.
-    if task["run"]["sparse-profile"]:
-        cache_name += "-sparse"
-
     return cache_name
 
 

template/{{cookiecutter.project_name}}/taskcluster.github.yml

@@ -222,7 +222,7 @@ tasks:
                                         ACTION_CALLBACK: '${action.cb_name}'
 
                           cache:
-                              "${trustDomain}-project-${project}-level-${level}-checkouts-sparse-v2": /builds/worker/checkouts
+                              "${trustDomain}-project-${project}-level-${level}-checkouts-v2": /builds/worker/checkouts
 
                           features:
                               taskclusterProxy: true

test/test_scripts_run_task.py

@@ -195,7 +195,6 @@ def test_collect_vcs_options(
 
     args.setdefault(f"{name}_checkout", checkout)
     args.setdefault(f"{name}_shallow_clone", False)
-    args.setdefault(f"{name}_sparse_profile", False)
     args = Namespace(**args)
 
     result = run_task_mod.collect_vcs_options(args, name, name)
@@ -214,7 +213,6 @@ def test_collect_vcs_options(
         "repo-type": env.get("REPOSITORY_TYPE"),
         "shallow-clone": False,
         "ssh-secret-name": env.get("SSH_SECRET_NAME"),
-        "sparse-profile": False,
         "store-path": env.get("HG_STORE_PATH"),
     }
     if "PIP_REQUIREMENTS" in env:

test/test_transforms_run_toolchain.py

@@ -65,7 +65,6 @@ def assert_docker_worker(task, taskdesc):
         "run": {
             "command": ["vcs/taskcluster/scripts/toolchain/run.sh"],
             "cwd": "{checkout}/..",
-            "sparse-profile": "toolchain-build",
             "using": "run-task",
             "workdir": "/builds/worker",
         },
@@ -140,7 +139,6 @@ def assert_generic_worker(task, taskdesc):
         "run": {
             "command": "src/taskcluster/scripts/toolchain/run.sh --foo bar",
             "cwd": "{checkout}/..",
-            "sparse-profile": "toolchain-build",
             "using": "run-task",
             "workdir": "/builds/worker",
         },
@@ -192,7 +190,6 @@ def assert_powershell(task, _):
         "command": "src/taskcluster/scripts/toolchain/run.ps1",
         "cwd": "{checkout}/..",
         "exec-with": "powershell",
-        "sparse-profile": "toolchain-build",
         "using": "run-task",
         "workdir": "/builds/worker",
     }