Add integration tests for redacting secrets from chat

tests/integration/anthropic/testcases.yaml

@@ -85,3 +85,23 @@ testcases:
       https://www.insight.stacklok.com/report/pypi/invokehttp?utm_source=codegate
     does_not_contain: |
       import invokehttp
+
+  anthropic_secret_redaction:
+    name: Anthropic Secret Redaction
+    provider: anthropic
+    url: http://127.0.0.1:8989/anthropic/messages
+    data: |
+      {
+        "messages": [
+          {
+            "role": "user",
+            "content":"Here's my API key: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789. Can you help me list my repos on GitHub?"
+          }
+        ],
+        "model":"claude-3-5-haiku-20241022",
+        "stream": true
+      }
+    contains: |
+      CodeGate prevented 1 secret
+    not_contains: |
+      ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789

tests/integration/copilot/testcases.yaml

@@ -11,6 +11,10 @@ testcases:
     data: |
       {
         "messages":[
+            {
+              "content":"You are a coding assistant.",
+              "role":"system"
+            },
             {
               "content":"Reply with that exact sentence: Hello from the integration tests!",
               "role":"user"
@@ -29,6 +33,10 @@ testcases:
     data: |
       {
         "messages":[
+            {
+              "content":"You are a coding assistant.",
+              "role":"system"
+            },
             {
               "content":"Generate me example code using the python invokehttp package to call an API",
               "role":"user"
@@ -42,3 +50,26 @@ testcases:
     does_not_contain: |
       import invokehttp
 
+  copilot_secret_redacting_chat:
+    name: Copilot secret redacting chat
+    provider: copilot
+    url: "https://api.openai.com/v1/chat/completions"
+    data: |
+      {
+        "messages":[
+            {
+              "content":"You are a coding assistant.",
+              "role":"system"
+            },
+            {
+              "content":"Here's my API key: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789. Can you help me list my repos on GitHub?",
+              "role":"user"
+            }
+        ],
+        "model":"gpt-4o-mini",
+        "stream":true
+      }
+    contains: |
+      CodeGate prevented 1 secret
+    not_contains: |
+      ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789

tests/integration/llamacpp/testcases.yaml

@@ -66,3 +66,29 @@ testcases:
       }
     contains: |
       CodeGate detected one or more malicious, deprecated or archived packages.
+
+  llamacpp_secret_redacting_chat:
+    name: LlamaCPP secret redacting chat
+    provider: llamacpp
+    url: http://127.0.0.1:8989/llamacpp/chat/completions
+    data: |
+      {
+        "max_tokens":4096,
+        "messages":[
+            {
+              "content":"You are a coding assistant.",
+              "role":"system"
+            },
+            {
+              "content":"Here's my API key: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789. Can you help me list my repos on GitHub?",
+              "role":"user"
+            }
+        ],
+        "model":"qwen2.5-coder-0.5b-instruct-q5_k_m",
+        "stream":true,
+        "temperature":0
+      }
+    contains: |
+      CodeGate prevented 1 secret
+    not_contains: |
+      ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789

tests/integration/ollama/testcases.yaml

@@ -86,3 +86,29 @@ testcases:
       }
     contains: |
       CodeGate detected one or more malicious, deprecated or archived packages.
+
+  ollama_secret_redacting_chat:
+    name: Ollama secret redacting chat
+    provider: ollama
+    url: http://127.0.0.1:8989/ollama/chat/completions
+    data: |
+      {
+        "max_tokens":4096,
+        "messages":[
+            {
+              "content":"You are a coding assistant.",
+              "role":"system"
+            },
+            {
+              "content":"Here's my API key: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789. Can you help me list my repos on GitHub?",
+              "role":"user"
+            }
+        ],
+        "model":"qwen2.5-coder:0.5b",
+        "stream":true,
+        "temperature":0
+      }
+    contains: |
+      CodeGate prevented 1 secret
+    not_contains: |
+      ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789

tests/integration/openai/testcases.yaml

@@ -78,3 +78,29 @@ testcases:
       https://www.insight.stacklok.com/report/pypi/invokehttp?utm_source=codegate
     does_not_contain: |
       import invokehttp
+
+  openai_secret_redacting_chat:
+    name: OpenAI secret redacting chat
+    provider: openai
+    url: http://127.0.0.1:8989/openai/chat/completions
+    data: |
+      {
+        "max_tokens":4096,
+        "messages":[
+            {
+              "content":"You are a coding assistant.",
+              "role":"system"
+            },
+            {
+              "content":"Here's my API key: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789. Can you help me list my repos on GitHub?",
+              "role":"user"
+            }
+        ],
+        "model":"gpt-4o-mini",
+        "stream":true,
+        "temperature":0
+      }
+    contains: |
+      CodeGate prevented 1 secret
+    not_contains: |
+      ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789

tests/integration/vllm/testcases.yaml

@@ -86,3 +86,30 @@ testcases:
       }
     contains: |
       CodeGate detected one or more malicious, deprecated or archived packages.
+
+# Commented out because it takes too much time in the CI pipeline. Also, this functionality should not be provider-specific
+#  vllm_secret_redacting_chat:
+#    name: VLLM secret redacting chat
+#    provider: vllm
+#    url: http://127.0.0.1:8989/vllm/chat/completions
+#    data: |
+#      {
+#        "max_tokens":4096,
+#        "messages":[
+#            {
+#              "content":"You are a coding assistant.",
+#              "role":"system"
+#            },
+#            {
+#              "content":"Here's my API key: ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789. Can you help me list my repos on GitHub?",
+#              "role":"user"
+#            }
+#        ],
+#        "model":"Qwen/Qwen2.5-Coder-0.5B-Instruct",
+#        "stream":true,
+#        "temperature":0
+#      }
+#    contains: |
+#      CodeGate prevented 1 secret
+#    not_contains: |
+#      ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789