chore: add pre-commit

.github/workflows/go.yml

@@ -34,7 +34,7 @@ jobs:
         go-version: '1.23'
 
     - name: Build
-      run: | 
+      run: |
         go run mage.go generate
 
     - name: Test

.github/workflows/python.yml

@@ -49,9 +49,7 @@ jobs:
 
       - name: "Generate parser files"
         run: |
-          pushd g4
           ./generate.sh
-          popd
 
       - name: Install dependencies
         run: |
@@ -60,4 +58,3 @@ jobs:
       - name: "Run unit tests"
         run: |
           uv run pytest -vs
-

.gitignore

@@ -74,4 +74,4 @@ go.work.sum
 
 # Editor/IDE
 .idea/
-.vscode/
+.vscode/

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+---
+ci:
+    autofix_commit_msg: |
+        chore(formatting): auto fixes from pre-commit hooks
+
+        for more information, see https://pre-commit.ci
+    autofix_prs: true
+    skip: []
+    submodules: false
+# Update the rev variable with the release version that you want, from the yamllint repo
+# You can pass your custom .yamllint with args attribute.
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+    - id: check-yaml
+      args: [--allow-multiple-documents]
+    - id: end-of-file-fixer
+    - id: trailing-whitespace
+      exclude: '^regex-assembly/'
+      args: [--markdown-linebreak-ext=md]
+- repo: local
+  hooks:
+    - id: run-generate
+      name: run-generate
+      entry: './generate.sh'
+      language: script
+      pass_filenames: false
+      files: '^g4/'

README.md

@@ -4,7 +4,7 @@ This repo contains the ANTLR files for a SecLang parser.
 
 ## Why a new parser?
 
-There has been efforts towards having parsers in different languages. Using ANTLR would allow us to have a common parser and generate parsing engines for different languages easily. 
+There has been efforts towards having parsers in different languages. Using ANTLR would allow us to have a common parser and generate parsing engines for different languages easily.
 This way we would consolidate efforts, and we can have a more robust parser.
 
 ## Features we (might) want

g4/SecLangLexer.g4

@@ -23,7 +23,7 @@ tokens {
 }
 
 WS
-   : ([ \t\r\n]+ | '\\' '\n' | '\\')  -> skip 
+   : ([ \t\r\n]+ | '\\' '\n' | '\\')  -> skip
    ;
 
 COMMENT
@@ -117,8 +117,8 @@ RPAREN
 // MODSEC CONFIG
 ACTION_ACCURACY
 	: 'accuracy'
-	; 
-	
+	;
+
 ACTION_ALLOW
 	: 'allow:' ('REQUEST'|'PHASE') | ('phase:' ('REQUEST|PHASE') | 'allow')
 	;
@@ -1256,11 +1256,11 @@ SPACE_COL
 	: ' ' -> skip, pushMode(OPERATOR_START_MODE)
 	;
 
-COMMA_COL 
+COMMA_COL
 	: ',' -> type(COMMA), popMode
 	;
 
-QUOTE_COL 
+QUOTE_COL
 	: '"' -> type(QUOTE), popMode
 	;
 
@@ -1274,11 +1274,11 @@ SPACE_VAR
 	: ' ' -> skip, pushMode(OPERATOR_START_MODE)
 	;
 
-COMMA_VAR 
+COMMA_VAR
 	: ',' -> type(COMMA), popMode
 	;
 
-QUOTE_VAR 
+QUOTE_VAR
 	: '"' -> type(QUOTE), popMode
 	;
 
@@ -1323,10 +1323,10 @@ NOT_OPERATOR
 	;
 
 SKIP_CHARS
-   : [\\\t\r\n ]+  -> skip 
+   : [\\\t\r\n ]+  -> skip
    ;
 
-QUOTE_OP 
+QUOTE_OP
 	: '"' -> type(QUOTE), pushMode(OPERATOR_WITH_QUOTES)
 	;
 
@@ -1347,4 +1347,3 @@ AT
 OPERATOR_QUOTED_STRING
     : (('\\"') | ~([" @!])) (('\\"')|~('"'))* -> pushMode(DEFAULT_MODE)
     ;
-

g4/SecLangParser.g4

@@ -487,4 +487,4 @@ assignment:
     EQUAL
     | EQUALS_PLUS
     | EQUALS_MINUS
-    ;
+    ;

g4/generate.go

@@ -3,4 +3,4 @@
 
 package g4
 
-//go:generate ./generate.sh
+//go:generate ../generate.sh

generate.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+# Copyright 2023 Felipe Zipitria
+# SPDX-License-Identifier: Apache-2.0
+
+# This script is used to generate the parser files for the seclang DSL.
+# It is used by the pre-commit hook to ensure that the parser files are up to date.
+
+# Check if java is installed
+if ! command -v java >/dev/null 2>&1; then
+    echo "Java is not installed. Please install Java and try again."
+    exit 1
+fi
+
+# Find g4 files and change directory
+g4_files=$(find . -name "SecLangLexer.g4")
+g4_dir=$(dirname "$g4_files")
+
+# Change directory to g4
+cd "$g4_dir"
+
+alias antlr4='java -Xmx500M -cp "../lib/antlr-4.13.2-complete.jar:$CLASSPATH" org.antlr.v4.Tool'
+antlr4 -Dlanguage=Go -no-visitor -package parser -o ../parser *.g4
+antlr4 -Dlanguage=Python3 -no-visitor -package parser -o ../src/seclang_parser *.g4

src/seclang_parser/actions.py

@@ -81,4 +81,4 @@ def get_actions_by_key(self, key):
         actions = [action for action in self.non_disruptive_actions if action.get_key() == key]
         actions.extend(action for action in self.flow_actions if action.get_key() == key)
         actions.extend(action for action in self.data_actions if action.get_key() == key)
-        return actions
+        return actions

src/seclang_parser/cli.py

@@ -34,4 +34,3 @@ def run():
 
 if __name__ == "__main__":
     run()
-

src/seclang_parser/configuration.py

@@ -8,4 +8,4 @@ def __init__(self):
         self.configurations = []
 
     def add_configuration(self, configuration):
-        self.configurations.append(configuration)
+        self.configurations.append(configuration)

src/seclang_parser/directives.py

@@ -155,4 +155,4 @@ def to_seclang_with_param(self, initial_string):
         result += "\n"
         if chained_rule and self.chained_rule:
             result += self.chained_rule.to_seclang_with_param(initial_string + "    ")
-        return result
+        return result

src/seclang_parser/metadata.py

@@ -88,4 +88,4 @@ def add_tag(self, value):
         self.tags.append(value)
 
     def set_ver(self, value):
-        self.ver = value
+        self.ver = value

src/seclang_parser/operators.py

@@ -10,4 +10,4 @@ def set_operator_value(self, value):
         self.value = value
 
     def to_string(self):
-        return f"@{self.name} {self.value}"
+        return f"@{self.name} {self.value}"

src/seclang_parser/parser.py

@@ -41,5 +41,3 @@ def enterComment(self, ctx: SecLangParser.CommentContext):
 
     def enterId(self, ctx:SecLangParser.IdContext):
         self.ids.append(ctx.getText())
-
-

src/seclang_parser/transformations.py

@@ -7,4 +7,4 @@ def add_transformation(self, transformation):
 
     def to_string(self):
         results = [f"t:{transformation}" for transformation in self.transformations]
-        return ",".join(results)
+        return ",".join(results)

src/seclang_parser/variables.py

@@ -6,4 +6,4 @@ def to_string(self):
         return "|".join(self.variables)
 
     def add_variable(self, variable):
-        self.variables.append(variable)
+        self.variables.append(variable)

testdata/REQUEST-905-COMMON-EXCEPTIONS.conf

@@ -30,4 +30,4 @@ SecRule REQUEST_LINE "@streq GET /" \
     SecRule REMOTE_ADDR "@ipMatch 127.0.0.1,::1" \
         "t:none,\
         ctl:ruleRemoveByTag=OWASP_CRS,\
-        ctl:auditEngine=Off"
+        ctl:auditEngine=Off"

testdata/plugins/drupal-rule-exclusions-before.conf

@@ -9,7 +9,7 @@
 
 # OWASP CRS Plugin
 # Plugin name: drupal-rule-exclusions
-# Plugin description: 
+# Plugin description:
 # Rule ID block base: 9,506,000 - 9,506,999
 # Plugin version: 1.0.0
 

testdata/plugins/drupal-rule-exclusions-config.conf

@@ -9,7 +9,7 @@
 
 # OWASP CRS Plugin
 # Plugin name: drupal-rule-exclusions
-# Plugin description: 
+# Plugin description:
 # Rule ID block base: 9,506,000 - 9,506,999
 # Plugin version: 1.0.0
 

testdata/plugins/google-oauth2-before.conf

@@ -9,7 +9,7 @@
 
 # OWASP CRS Plugin
 # Plugin name: google-oauth2
-# Plugin description: 
+# Plugin description:
 # Rule ID block base: 9,505,000 - 9,505,999
 # Plugin version: 1.0.0
 

testdata/plugins/google-oauth2-config.conf

@@ -9,7 +9,7 @@
 
 # OWASP CRS Plugin
 # Plugin name: google-oauth2
-# Plugin description: 
+# Plugin description:
 # Rule ID block base: 9,505,000 - 9,505,999
 # Plugin version: 1.0.0
 

testdata/plugins/wordpress-rule-exclusions-config.conf

@@ -9,7 +9,7 @@
 
 # OWASP CRS Plugin
 # Plugin name: wordpress-rule-exclusions
-# Plugin description: 
+# Plugin description:
 # Rule ID block base: 9,507,000 - 9,507,999
 # Plugin version: 1.0.1
 

testdata/test6.conf

@@ -10,4 +10,4 @@ SecRule TX:enforce_bodyproc_urlencoded "@eq 1" \
     ver:'OWASP_CRS/4.0.0-rc1',\
     chain"
     SecRule REQBODY_PROCESSOR "!@rx (?:URLENCODED|MULTIPART|XML|JSON)" \
-        "ctl:requestBodyProcessor=URLENCODED"
+        "ctl:requestBodyProcessor=URLENCODED"

testdata/test7.conf

@@ -41,10 +41,10 @@ SecRule TX:enforce_bodyproc_urlencoded "@unconditionalMatch" "id:200, phase:2, d
 SecRule FILES "@rx \.conf$" "id:17"
 
 
-# Detect Nikto 
+# Detect Nikto
 SecRule REQUEST_HEADERS:User-Agent "@rx nikto" "phase:1,id:173,t:lowercase"
 
-# Detect Nikto with a case-insensitive pattern 
+# Detect Nikto with a case-insensitive pattern
 SecRule REQUEST_HEADERS:User-Agent "@rx (?i)nikto" "phase:1,id:174,t:none"
 
 
@@ -54,4 +54,4 @@ SecAction \
     pass,\
     t:none,\
     nolog,\
-    setvar:tx.crs_setup_version=400"
+    setvar:tx.crs_setup_version=400"

testdata/test_04_directives.conf

@@ -7,4 +7,4 @@ SecUnicodeMapFile unicode.mapping 20127
 
 SecAuditLog /path/to/audit.log
 
-SecAuditLog "/path/to/audit.log"
+SecAuditLog "/path/to/audit.log"

testdata/test_06_secaction2.conf

@@ -19,4 +19,3 @@ SecAction "id:900005,\
     setvar:tx.arg_length=400,\
     setvar:tx.max_file_size=64100,\
     setvar:tx.combined_file_sizes=65535"
-

testdata/test_07_secaction3.conf

@@ -19,4 +19,3 @@ SecAction "id:900005,\
     setvar:'tx.arg_length=400',\
     setvar:tx.max_file_size=64100,\
     setvar:tx.combined_file_sizes=65535"
-

testdata/test_14_secaction_ctl_06.conf

@@ -5,4 +5,3 @@ SecAction "id:900005,\
     pass,\
     ctl:ruleEngine=DetectionOnly,\
     ctl:ruleRemoveById=910000"
-

testdata/test_15_secaction_01.conf

@@ -11,4 +11,3 @@ SecAction "id:900005,\
     setvar:'tx.arg_length=400',\
     setvar:tx.max_file_size=64100,\
     setvar:tx.combined_file_sizes=65535"
-

testdata/test_18_secrule_03.conf

@@ -1,2 +1,2 @@
 # Expect a failure on non-existent operator
-SecRule REQUEST_HEADERS:X-CRS-Test "@ry foo" 
+SecRule REQUEST_HEADERS:X-CRS-Test "@ry foo"

testdata/test_25_secrule_10.conf

@@ -17,5 +17,3 @@ SecRule FILES|FILES_NAMES "!@rx (?i)^(?:&(?:(?:[acegiln-or-suz]acut|[aeiou]grav|
     ver:'OWASP_CRS/4.0.0-rc2',\
     severity:'CRITICAL',\
     setvar:'tx.inbound_anomaly_score_pl1=+%{tx.critical_anomaly_score}'"
-
-

testdata/test_26_secrule_11.conf

@@ -20,4 +20,3 @@ SecRule REQUEST_URI "@rx \x25" \
     chain"
     SecRule REQUEST_URI "@validateUrlEncoding" \
         "setvar:'tx.inbound_anomaly_score_pl1=+%{tx.warning_anomaly_score}'"
-

testdata/test_27_secrule_12.conf

@@ -1,4 +1,3 @@
 
 SecRule REQUEST_URI "@validateUrlEncoding" \
     "setvar:'tx.inbound_anomaly_score_pl1=+%{tx.warning_anomaly_score}'"
-

testdata/test_28_secrule_13.conf

@@ -21,4 +21,3 @@ SecRule FILES|REQUEST_HEADERS:X-Filename|REQUEST_HEADERS:X_Filename|REQUEST_HEAD
     severity:'CRITICAL',\
     setvar:'tx.rce_score=+%{tx.critical_anomaly_score}',\
     setvar:'tx.inbound_anomaly_score_pl1=+%{tx.critical_anomaly_score}'"
-

testdata/test_29_secrule_14.conf

@@ -10,4 +10,3 @@ SecRule REQUEST_FILENAME "!@validateByteRange 20, 45-47, 48-57, 65-90, 95, 97-12
 
 
 #
-

testdata/test_32_secrule_16.conf

@@ -18,4 +18,3 @@ SecRule TX:DO_REPUT_BLOCK "@eq 1" \
         "setvar:'tx.anomaly_score_pl1=+%{tx.critical_anomaly_score}'"
 
 #
-

testdata/test_33_secrule_16.conf

@@ -21,4 +21,3 @@ SecRule REQUEST_URI|REQUEST_HEADERS|ARGS|ARGS_NAMES "@rx (?:^|[^\\\\])\\\\[cdegh
 
 
 #
-

testdata/test_35_all_directives.conf

@@ -39,4 +39,3 @@ SecRule TX:enforce_bodyproc_urlencoded "@eq 1" \
     noauditlog,\
     msg:'Enabling forced body inspection for ASCII content',\
     ver:'OWASP_CRS/4.0.0-rc1'"
-

testdata/test_36_chain.conf

@@ -56,4 +56,3 @@ SecRule REMOTE_ADDR "@ipMatch 127.0.0.1,::1" \
             "t:none,\
             ctl:ruleRemoveByTag=OWASP_CRS,\
             ctl:auditEngine=Off"
-