Log error for non-default analysis-kinds input outside of managed workflows

Author: mbg

lib/init-action.js

@@ -16,7 +16,12 @@ import {
 } from "./analyses";
 import { EnvVar } from "./environment";
 import { getRunnerLogger } from "./logging";
-import { createFeatures, RecordingLogger, setupTests } from "./testing-utils";
+import {
+  createFeatures,
+  RecordingLogger,
+  setupBaseActionsVars,
+  setupTests,
+} from "./testing-utils";
 import { AssessmentPayload } from "./upload-lib/types";
 import { ConfigurationError } from "./util";
 
@@ -72,6 +77,7 @@ test.serial(
 test.serial(
   "getAnalysisKinds - only use `code-scanning` for multiple analysis kinds outside of test mode",
   async (t) => {
+    setupBaseActionsVars();
     process.env[EnvVar.TEST_MODE] = "false";
     const features = createFeatures([]);
     const logger = new RecordingLogger();
@@ -89,6 +95,44 @@ test.serial(
   },
 );
 
+test.serial(
+  "getAnalysisKinds - logs error for non-default `analysis-kinds` in custom workflow",
+  async (t) => {
+    setupBaseActionsVars({ GITHUB_EVENT_NAME: "push" });
+    process.env[EnvVar.TEST_MODE] = "false";
+    const features = createFeatures([]);
+    const logger = new RecordingLogger();
+    const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
+    requiredInputStub.withArgs("analysis-kinds").returns("code-quality");
+    const result = await getAnalysisKinds(logger, features, true);
+    t.deepEqual(result, [AnalysisKind.CodeQuality]);
+    t.assert(
+      logger.hasMessage(
+        "An analysis kind other than `code-scanning` was specified in a custom workflow.",
+      ),
+    );
+  },
+);
+
+test.serial(
+  "getAnalysisKinds - no error for non-default `analysis-kinds` in managed workflow",
+  async (t) => {
+    setupBaseActionsVars({ GITHUB_EVENT_NAME: "dynamic" });
+    process.env[EnvVar.TEST_MODE] = "false";
+    const features = createFeatures([]);
+    const logger = new RecordingLogger();
+    const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
+    requiredInputStub.withArgs("analysis-kinds").returns("code-quality");
+    const result = await getAnalysisKinds(logger, features, true);
+    t.deepEqual(result, [AnalysisKind.CodeQuality]);
+    t.assert(
+      !logger.hasMessage(
+        "An analysis kind other than `code-scanning` was specified in a custom workflow.",
+      ),
+    );
+  },
+);
+
 test.serial(
   "getAnalysisKinds - includes `code-quality` when deprecated `quality-queries` input is used",
   async (t) => {
@@ -133,6 +177,7 @@ for (let i = 0; i < analysisKinds.length; i++) {
       test.serial(
         `getAnalysisKinds - allows ${analysisKind} with ${otherAnalysis}`,
         async (t) => {
+          setupBaseActionsVars();
           process.env[EnvVar.TEST_MODE] = "true";
           const features = createFeatures([]);
           const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
@@ -151,6 +196,7 @@ for (let i = 0; i < analysisKinds.length; i++) {
       test.serial(
         `getAnalysisKinds - throws if ${analysisKind} is enabled with ${otherAnalysis}`,
         async (t) => {
+          setupBaseActionsVars();
           const features = createFeatures([]);
           const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
           requiredInputStub

lib/setup-codeql-action.js

@@ -2,6 +2,7 @@ import {
   fixCodeQualityCategory,
   getOptionalInput,
   getRequiredInput,
+  isDynamicWorkflow,
 } from "./actions-util";
 import { EnvVar } from "./environment";
 import { Feature, FeatureEnablement } from "./feature-flags";
@@ -65,6 +66,21 @@ export async function parseAnalysisKinds(
 // Used to avoid re-parsing the input after we have done it once.
 let cachedAnalysisKinds: AnalysisKind[] | undefined;
 
+/** Determines whether `code-scanning` is the only enabled analysis kind in `analysisKinds`. */
+function isOnlyCodeScanningEnabled(analysisKinds: AnalysisKind[]) {
+  return (
+    analysisKinds.length === 1 && analysisKinds[0] === AnalysisKind.CodeScanning
+  );
+}
+
+/** Prepends a generic message about the intended usage for `analysis-kinds` to `message`. */
+function makeAnalysisKindUsageError(message: string) {
+  return (
+    "The `analysis-kinds` input is experimental and for GitHub-internal use only. " +
+    `Its behaviour may change at any time or be removed entirely. ${message}`
+  );
+}
+
 /**
  * Initialises the analysis kinds for the analysis based on the `analysis-kinds` input.
  * This function will also use the deprecated `quality-queries` input as an indicator to enable `code-quality`.
@@ -89,6 +105,26 @@ export async function getAnalysisKinds(
     getRequiredInput("analysis-kinds"),
   );
 
+  // Log an error if we are outside of a GitHub-managed workflow and an analysis kind
+  // other than `code-scanning` is enabled.
+  if (
+    !isInTestMode() &&
+    !isDynamicWorkflow() &&
+    !isOnlyCodeScanningEnabled(analysisKinds)
+  ) {
+    const codeQualityHint = analysisKinds.includes(AnalysisKind.CodeQuality)
+      ? " If your intention is to use quality queries outside of Code Quality, " +
+        "use the `queries` input with `code-quality` instead."
+      : "";
+
+    logger.error(
+      makeAnalysisKindUsageError(
+        "An analysis kind other than `code-scanning` was specified in a custom workflow. " +
+          `This is not supported and will become a fatal error in a future version of the CodeQL Action.${codeQualityHint}`,
+      ),
+    );
+  }
+
   // Warn that `quality-queries` is deprecated if there is an argument for it.
   const qualityQueriesInput = getOptionalInput("quality-queries");
 
@@ -130,10 +166,10 @@ export async function getAnalysisKinds(
     !(await features.getValue(Feature.AllowMultipleAnalysisKinds))
   ) {
     logger.error(
-      "The `analysis-kinds` input is experimental and for GitHub-internal use only. " +
-        "Its behaviour may change at any time or be removed entirely. " +
+      makeAnalysisKindUsageError(
         "Specifying multiple values as input is no longer supported. " +
-        "Continuing with only `analysis-kinds: code-scanning`.",
+          "Continuing with only `analysis-kinds: code-scanning`.",
+      ),
     );
 
     // Only enable Code Scanning.

src/analyses.test.ts

@@ -188,17 +188,37 @@ export const DEFAULT_ACTIONS_VARS = {
   RUNNER_OS: "Linux",
 } as const satisfies Record<string, string>;
 
-// Sets environment variables that make using some libraries designed for
-// use only on actions safe to use outside of actions.
-export function setupActionsVars(
-  tempDir: string,
-  toolsDir: string,
-  overrides?: Partial<Record<keyof typeof DEFAULT_ACTIONS_VARS, string>>,
-) {
+/** Partial mappings from GitHub Actions environment variables to values. */
+export type ActionVarOverrides = Partial<
+  Record<keyof typeof DEFAULT_ACTIONS_VARS, string>
+>;
+
+/**
+ * Sets environment variables that are always available on GitHub Actions,
+ * excluding some that are expected to be set to paths. See `setupActionsVars`.
+ *
+ * @param overrides Overrides for the defaults.
+ */
+export function setupBaseActionsVars(overrides?: ActionVarOverrides) {
   const vars = { ...DEFAULT_ACTIONS_VARS, ...overrides };
   for (const [key, value] of Object.entries(vars)) {
     process.env[key] = value;
   }
+}
+
+/**
+ * Sets environment variables that are always available on GitHub Actions.
+ *
+ * @param tempDir A value for `RUNNER_TEMP` and `GITHUB_WORKSPACE`.
+ * @param toolsDir A value for `RUNNER_TOOL_CACHE`.
+ * @param overrides Overrides for the defaults.
+ */
+export function setupActionsVars(
+  tempDir: string,
+  toolsDir: string,
+  overrides?: ActionVarOverrides,
+) {
+  setupBaseActionsVars(overrides);
   process.env["RUNNER_TEMP"] = tempDir;
   process.env["RUNNER_TOOL_CACHE"] = toolsDir;
   process.env["GITHUB_WORKSPACE"] = tempDir;