Merge pull request #36 from olehermanse/testing

olehermanse · web-flow · commit fe3706840388 · 2026-03-30T23:40:46.000+02:00
Added linting tests
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -47,3 +47,7 @@ jobs:
         run: |
           source .venv/bin/activate
           bash tests/run-format-tests.sh
+      - name: Lint tests
+        run: |
+          source .venv/bin/activate
+          bash tests/run-lint-tests.sh
diff --git a/tests/README.md b/tests/README.md
@@ -0,0 +1,58 @@
+# Tests
+
+There are 4 types of tests: unit, shell, format, and lint.
+All test scripts should be run from the repository root.
+
+## Unit tests
+
+Python unit tests using pytest, located in `tests/unit/` as `test_*.py` files.
+
+```bash
+uv run pytest
+```
+
+To add a new test, create a `tests/unit/test_<name>.py` file with test functions prefixed `test_`.
+
+## Shell tests
+
+Bash scripts in `tests/shell/` that exercise the CLI end-to-end.
+Each script is a standalone test that runs commands and relies on `set -e` to fail on errors.
+
+```bash
+bash tests/run-shell-tests.sh
+```
+
+To add a new test, create a `tests/shell/<NNN>-<name>.sh` file.
+
+## Format tests
+
+Tests for `cfengine format`, located in `tests/format/`.
+Each test is a pair of files:
+
+- `<name>.input.cf` - the unformatted input
+- `<name>.expected.cf` - the expected formatted output
+
+The test runner formats each input file and diffs against the expected output.
+
+```bash
+bash tests/run-format-tests.sh
+```
+
+To add a new test, create both an `.input.cf` and `.expected.cf` file with matching names.
+
+## Lint tests
+
+Tests for `cfengine lint`, located in `tests/lint/`.
+Each test is a `.cf` file.
+The file extension determines the expected outcome:
+
+- `<name>.cf` - expected to pass linting (exit code 0)
+- `<name>.x.cf` - expected to fail linting (non-zero exit code), must have a corresponding `<name>.output.txt` file containing the expected error output
+
+```bash
+bash tests/run-lint-tests.sh
+```
+
+To add a passing test, create a `tests/lint/<NNN>_<name>.cf` file.
+To add a failing test, create both a `tests/lint/<NNN>_<name>.x.cf` file and a `tests/lint/<NNN>_<name>.output.txt` file with the expected lint output.
+The output file must match exactly, including the relative file path (e.g. `tests/lint/<NNN>_<name>.x.cf`).
diff --git a/tests/lint/001_hello_world.cf b/tests/lint/001_hello_world.cf
@@ -0,0 +1,5 @@
+bundle agent main
+{
+  reports:
+    "Hello, world!";
+}
diff --git a/tests/lint/002_ifvarclass.output.txt b/tests/lint/002_ifvarclass.output.txt
@@ -0,0 +1,7 @@
+
+    "Hello, CFEngine"
+      ifvarclass => "cfengine";
+      ^--------^
+Deprecation: Use 'if' instead of 'ifvarclass' at tests/lint/002_ifvarclass.x.cf:5:7
+FAIL: tests/lint/002_ifvarclass.x.cf (1 errors)
+Failure, 1 errors in total.
diff --git a/tests/lint/002_ifvarclass.x.cf b/tests/lint/002_ifvarclass.x.cf
@@ -0,0 +1,6 @@
+bundle agent main
+{
+  reports:
+    "Hello, CFEngine"
+      ifvarclass => "cfengine";
+}
diff --git a/tests/lint/003_deprecated_promise_type.cf b/tests/lint/003_deprecated_promise_type.cf
@@ -0,0 +1,5 @@
+bundle agent main
+{
+  vars:
+    "x" string => "value";
+}
diff --git a/tests/lint/003_deprecated_promise_type.output.txt b/tests/lint/003_deprecated_promise_type.output.txt
@@ -0,0 +1,7 @@
+
+{
+  defaults:
+  ^-------^
+Deprecation: Promise type 'defaults' is deprecated at tests/lint/003_deprecated_promise_type.x.cf:3:3
+FAIL: tests/lint/003_deprecated_promise_type.x.cf (1 errors)
+Failure, 1 errors in total.
diff --git a/tests/lint/003_deprecated_promise_type.x.cf b/tests/lint/003_deprecated_promise_type.x.cf
@@ -0,0 +1,5 @@
+bundle agent main
+{
+  defaults:
+    "x" string => "value";
+}
diff --git a/tests/lint/004_bundle_name_lowercase.cf b/tests/lint/004_bundle_name_lowercase.cf
@@ -0,0 +1,5 @@
+bundle agent my_bundle
+{
+  reports:
+    "Hello";
+}
diff --git a/tests/lint/004_bundle_name_lowercase.output.txt b/tests/lint/004_bundle_name_lowercase.output.txt
@@ -0,0 +1,6 @@
+
+bundle agent MyBundle
+             ^------^
+Convention: Bundle name should be lowercase at tests/lint/004_bundle_name_lowercase.x.cf:1:14
+FAIL: tests/lint/004_bundle_name_lowercase.x.cf (1 errors)
+Failure, 1 errors in total.
diff --git a/tests/lint/004_bundle_name_lowercase.x.cf b/tests/lint/004_bundle_name_lowercase.x.cf
@@ -0,0 +1,5 @@
+bundle agent MyBundle
+{
+  reports:
+    "Hello";
+}
diff --git a/tests/lint/005_bundle_type.cf b/tests/lint/005_bundle_type.cf
@@ -0,0 +1,5 @@
+bundle common my_bundle
+{
+  vars:
+    "x" string => "value";
+}
diff --git a/tests/lint/005_bundle_type.output.txt b/tests/lint/005_bundle_type.output.txt
@@ -0,0 +1,6 @@
+
+bundle notavalidtype my_bundle
+       ^-----------^
+Error: Bundle type must be one of (agent, common, monitor, server, edit_line, edit_xml), not 'notavalidtype' at tests/lint/005_bundle_type.x.cf:1:8
+FAIL: tests/lint/005_bundle_type.x.cf (1 errors)
+Failure, 1 errors in total.
diff --git a/tests/lint/005_bundle_type.x.cf b/tests/lint/005_bundle_type.x.cf
@@ -0,0 +1,5 @@
+bundle notavalidtype my_bundle
+{
+  reports:
+    "Hello";
+}
diff --git a/tests/lint/006_syntax_error.cf b/tests/lint/006_syntax_error.cf
@@ -0,0 +1,5 @@
+bundle agent main
+{
+  reports:
+    "Valid syntax";
+}
diff --git a/tests/lint/006_syntax_error.output.txt b/tests/lint/006_syntax_error.output.txt
@@ -0,0 +1,7 @@
+
+{
+  reports
+  ^-----^
+Error: Syntax error at tests/lint/006_syntax_error.x.cf:3:3
+FAIL: tests/lint/006_syntax_error.x.cf (1 errors)
+Failure, 1 errors in total.
diff --git a/tests/lint/006_syntax_error.x.cf b/tests/lint/006_syntax_error.x.cf
@@ -0,0 +1,5 @@
+bundle agent main
+{
+  reports
+    "Missing colon after promise type";
+}
diff --git a/tests/lint/007_empty_file.output.txt b/tests/lint/007_empty_file.output.txt
@@ -0,0 +1,3 @@
+Error: Empty policy file 'tests/lint/007_empty_file.x.cf'
+FAIL: tests/lint/007_empty_file.x.cf (1 errors)
+Failure, 1 errors in total.
diff --git a/tests/lint/007_empty_file.x.cf b/tests/lint/007_empty_file.x.cf
diff --git a/tests/run-lint-tests.sh b/tests/run-lint-tests.sh
@@ -0,0 +1,45 @@
+#/usr/bin/env bash
+
+set -e
+# set -x
+
+echo "These tests expect cfengine CLI to be installed globally or in venv"
+
+echo "Looking for CFEngine CLI:"
+which cfengine
+
+echo "Check that test files are in expected location:"
+ls -al tests/lint/*.cf
+
+mkdir -p tmp
+
+echo "Run lint tests:"
+for file in tests/lint/*.cf; do
+  if echo "$file" | grep -q '\.x\.cf$'; then
+    # File ends with .x.cf, we expect it to:
+    #  - Fail (non-zero exit code)
+    #  - Output the correct error message
+
+    expected="$(echo $file | sed 's/\.x\.cf$/.output.txt/')"
+    if [ ! -f "$expected" ]; then
+      echo "FAIL: Missing expected output file: $expected"
+      exit 1
+    fi
+    output="tmp/$(basename $file .x.cf).lint-output.txt"
+    if cfengine lint "$file" > "$output" 2>&1; then
+      echo "FAIL: $file - expected lint failure but got success"
+      exit 1
+    fi
+    diff -u "$expected" "$output"
+    echo "OK (expected failure): $file"
+  else
+    # Expect success
+    if ! cfengine lint "$file"; then
+      echo "FAIL: $file - expected lint success but got failure"
+      exit 1
+    fi
+    echo "OK: $file"
+  fi
+done
+
+echo "All lint tests successful!"
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
diff --git a/tests/unit/test_deps.py b/tests/unit/test_deps.py
diff --git a/tests/unit/test_paths.py b/tests/unit/test_paths.py
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
diff --git a/tests/unit/test_version.py b/tests/unit/test_version.py

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +bundle agent main
 +{
 +  vars:
 +    "x" string => "value";
 +}