Skip to content

Commit 9e31fb5

Browse files
redsun82redsun82
committed
Just: fix and add windows
1 parent 2dea9da commit 9e31fb5

File tree

8 files changed

+151
-51
lines changed

8 files changed

+151
-51
lines changed

‎misc/just/codeql-test-run.ts‎

Lines changed: 81 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,97 @@
11
import * as child_process from "child_process";
22
import * as path from "path";
3+
import * as os from "os";
4+
5+
6+
7+
function invoke(invocation: string[], options: {cwd?: string, log_prefix?: string} = {}) : number {
8+
const log_prefix = options.log_prefix && options.log_prefix !== "" ? `${options.log_prefix} ` : "";
9+
console.log(`${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`);
10+
try {
11+
child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: options.cwd });
12+
} catch (error) {
13+
return 1;
14+
}
15+
return 0;
16+
}
17+
18+
type Args = {
19+
tests: string[];
20+
flags: string[];
21+
env: string[];
22+
build: boolean;
23+
testing_level: number;
24+
};
25+
26+
function parseArgs(args: Args, argv: string) {
27+
argv
28+
.split(/(?<!\\) /)
29+
.forEach((arg) => {
30+
if (arg === "--no-build") {
31+
args.build = false;
32+
} else if (arg.startsWith("-")) {
33+
args.flags.push(arg);
34+
} else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) {
35+
args.env.push(arg);
36+
} else if (/^\++$/.test(arg)) {
37+
args.testing_level = Math.max(args.testing_level, arg.length);
38+
} else if (arg !== "") {
39+
args.tests.push(arg);
40+
}
41+
});
42+
}
43+
344

445
function codeqlTestRun(argv: string[]): number {
5-
const [language, args, ...plus] = argv;
46+
const [language, extra_args, ...plus] = argv;
647
let codeql =
748
process.env["SEMMLE_CODE"] ?
849
path.join(process.env["SEMMLE_CODE"], "target", "intree", `codeql-${language}`, "codeql")
950
:
1051
"codeql"
1152
;
12-
process.env["CODEQL_CONFIG_FILE"] ||= "." // disable the default implicit config file, but keep an explicit one
13-
let plus_options = plus.map(option => option.trim().split("\n").filter(option => option !== ""));
14-
let testing_level = 0;
15-
let parsed_args = args.split(" ").filter(arg => {
16-
if (arg === "") return false;
17-
if (/^\++$/.test(arg)) {
18-
testing_level = Math.max(testing_level, arg.length);
19-
return false;
20-
}
21-
return true;
22-
});
23-
if (parsed_args.every(arg => arg.startsWith("-"))) {
24-
parsed_args.push(".");
53+
const ram_per_thread = process.platform === "linux" ? 3000 : 2048;
54+
const cpus = os.cpus().length;
55+
let args: Args = {
56+
tests: [],
57+
flags: [
58+
`--ram=${ram_per_thread}`,
59+
`-j${cpus}`,
60+
],
61+
env: [],
62+
build: true,
63+
testing_level: 0
64+
};
65+
parseArgs(args, extra_args);
66+
for (let i = 0; i < Math.min(plus.length, args.testing_level); i++) {
67+
parseArgs(args, plus[i]);
2568
}
26-
let invocation = [codeql, "test", "run", "-j0", ...parsed_args];
27-
for (let i = 0; i < Math.min(plus_options.length, testing_level); i++) {
28-
invocation.push(...plus_options[i]);
69+
if (args.tests.length === 0) {
70+
args.tests.push(".");
2971
}
30-
console.log(`${process.env["CMD_BEGIN"] || ""}${invocation.join(" ")}${process.env["CMD_END"] || ""}`);
31-
try {
32-
child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit" });
33-
} catch (error) {
34-
return 1;
72+
if (args.build && process.env["SEMMLE_CODE"]) {
73+
// If SEMMLE_CODE is set, we are in the semmle-code repo, so we build the codeql binary.
74+
// Otherwise, we use codeql from PATH.
75+
if (invoke(["python3", "build", `target/intree/codeql-${language}`], {cwd: process.env["SEMMLE_CODE"]}) !== 0) {
76+
return 1;
77+
}
3578
}
36-
return 0;
79+
process.env["CODEQL_CONFIG_FILE"] ||= "." // disable the default implicit config file, but keep an explicit one
80+
// Set and unset environment variables
81+
args.env.forEach((envVar) => {
82+
const [key, value] = envVar.split("=", 2);
83+
if (key) {
84+
if (value === undefined) {
85+
delete process.env[key];
86+
} else {
87+
process.env[key] = value;
88+
}
89+
} else {
90+
console.error(`Invalid environment variable assignment: ${envVar}`);
91+
process.exit(1);
92+
}
93+
});
94+
return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], {log_prefix: args.env.join(" ")});
3795
}
3896

3997
process.exit(codeqlTestRun(process.argv.slice(2)));

‎misc/just/forward-command.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,10 @@ function forwardCommand(args: string[]): number {
3636
}
3737
process.env[envVariable] = "true";
3838
const cmdArgs = args.slice(1);
39-
const is_flag = /^(-.*|\++)$/; // + is used for testing level in some langauge tests
40-
const flags = cmdArgs.filter(arg => is_flag.test(arg));
41-
const positionalArgs = cmdArgs.filter(arg => !is_flag.test(arg));
39+
// non-positional arguments are flags, repeated + (used by language tests) or environment variable settings
40+
const is_non_positional = /^(-.*|\++|[A-Z_][A-Z_0-9]*=.*)$/;
41+
const flags = cmdArgs.filter(arg => is_non_positional.test(arg));
42+
const positionalArgs = cmdArgs.filter(arg => !is_non_positional.test(arg));
4243

4344
if (positionalArgs.length === 0) {
4445
console.error("No positional arguments provided");

‎misc/just/lib.just‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,25 +28,31 @@ _build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE)
2828

2929
[no-exit-message]
3030
_maybe_build LANGUAGE:
31-
{{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd $SEMMLE_CODE; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }}
31+
{{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }}
3232

33-
[no-cd, positional-arguments, no-exit-message]
34-
@_codeql_test LANGUAGE +ARGS: (_maybe_build LANGUAGE)
35-
{{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@"
33+
34+
default_db_checks := """\
35+
--check-databases \
36+
--check-diff-informed \
37+
--fail-on-trap-errors \
38+
"""
3639

3740
[no-cd, positional-arguments, no-exit-message]
38-
@_codeql_test_run_only LANGUAGE +ARGS:
41+
@_codeql_test LANGUAGE +ARGS:
3942
{{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@"
4043

4144

4245
[no-cd, no-exit-message]
4346
_ql_format +ARGS: (_maybe_build "nolang")
44-
{{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/target/intree/codeql-nolang/' } else { '' } }}codeql query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }}
47+
{{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' } else { 'codeql' } }} query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }}
4548

4649

4750
[no-cd, no-exit-message]
4851
_bazel COMMAND *ARGS:
49-
{{ cmd_sep }}{{ if SEMMLE_CODE != "" { '$SEMMLE_CODE/tools/' } else { '' } }}bazel {{ COMMAND }} {{ ARGS }}{{ cmd_sep }}
52+
{{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/tools/bazel"' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }}
53+
54+
[no-cd, no-exit-message]
55+
_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS)
5056

5157
[no-cd, no-exit-message]
5258
_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS)
@@ -55,9 +61,14 @@ _integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest"
5561
_run +ARGS:
5662
{{ cmd_sep }}{{ ARGS }}{{ cmd_sep }}
5763

64+
[no-cd]
65+
_run_in_semmle_code +ARGS: _require_semmle_code
66+
{{ cmd_sep }}cd "$SEMMLE_CODE"; {{ ARGS }}{{ cmd_sep }}
67+
68+
5869
[no-cd, positional-arguments, no-exit-message]
5970
_just +ARGS:
60-
{{ JUST_EXECUTABLE }} "$@"
71+
"{{ JUST_EXECUTABLE }}" "$@"
6172

6273
[no-cd]
6374
_black *ARGS=".": (_run "uv" "run" "black" ARGS)

‎rust/ql/justfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,10 @@ import "../../misc/just/lib.just"
22

33
[no-cd]
44
format *ARGS=".": (_ql_format ARGS)
5+
6+
db_checks := default_db_checks
7+
8+
consistency := "--consistency-queries=" + source_dir() / "consistency-queries"
9+
10+
[no-cd]
11+
test *ARGS=".": (_codeql_test "rust" ARGS db_checks consistency)

‎rust/ql/test/justfile‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎swift/justfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import '../misc/just/lib.just'
2+
3+
install: (_bazel "run" "@codeql//swift:install")
4+
5+
build: (_build "swift")
6+
7+
generate: (_bazel "run" "@codeql//swift/codegen")
8+
9+
@_check_clang_format:
10+
if ! which clang-format > /dev/null; then \
11+
"{{ JUST_EXECUTABLE }}" _run_in_semmle_code "tools/bazel" "run" "//c/clang-format:install"; \
12+
fi
13+
14+
format ARGS=".": _check_clang_format (_run "clang-format" "-i" ("$(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')"))
15+

‎swift/ql/integration-tests/justfile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import "../../../misc/just/lib.just"
2+
3+
4+
[no-cd]
5+
test *ARGS=".": (_just "generate") (_integration_test ARGS)
6+
7+
# TODO in separate PR
8+
# [no-cd]
9+
# format *ARGS=".": (_ql_format ARGS) (_black ARGS)

‎swift/ql/justfile‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import "../../misc/just/lib.just"
2+
3+
[no-cd]
4+
format *ARGS=".": (_ql_format ARGS)
5+
6+
db_checks := default_db_checks + """\
7+
--check-repeated-labels \
8+
--check-redefined-labels \
9+
--check-use-before-definition \
10+
--consistency-queries=ql/swift/ql/consistency-queries \
11+
"""
12+
13+
consistency := "--consistency-queries=" + source_dir() / "consistency-queries"
14+
15+
[no-cd]
16+
test *ARGS=".": (_codeql_test "swift" ARGS db_checks consistency)

0 commit comments

Comments
 (0)