Merge pull request #98 from rootcodelabs/wip

Pulling changes from wip to dev

Author: Thirunayan22

.dockerignore

@@ -0,0 +1,106 @@
+# Docker ignore file for LLM Orchestration Service
+# Exclude unnecessary files from Docker build context
+
+# Git
+.git
+.gitignore
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Logs
+*.log
+logs/
+*.log.*
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+coverage.xml
+
+# Documentation
+docs/
+*.md
+!README.md
+
+# Config files (will be mounted)
+.env.local
+.env.development
+.env.test
+
+# Cache directories
+.ruff_cache/
+.mypy_cache/
+.pyright_cache/
+
+# Test files
+test_*.py
+*_test.py
+tests/
+
+# Development scripts
+run_*.py
+test_*.py
+
+# Temporary files
+*.tmp
+*.temp
+.temporary
+
+# Node modules (if any)
+node_modules/
+
+# Docker files (except the specific one being built)
+Dockerfile*
+!Dockerfile.llm_orchestration_service
+docker-compose*.yml
+
+# Grafana configs (not needed for this service)
+grafana-configs/

.env.example

@@ -0,0 +1,123 @@
+name: DeepEval RAG System Tests
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - '.github/workflows/deepeval-tests.yml'
+
+jobs:
+  deepeval-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: '.python-version'
+          
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v6
+        
+      - name: Install dependencies (locked)
+        run: uv sync --frozen
+        
+      - name: Run DeepEval tests
+        id: run_tests
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: uv run python -m pytest tests/deepeval_tests/standard_tests.py -v --tb=short
+          
+      - name: Generate evaluation report
+        if: always()
+        run: python tests/deepeval_tests/report_generator.py
+          
+      - name: Comment PR with test results
+        if: always() && github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            
+            try {
+              const reportContent = fs.readFileSync('test_report.md', 'utf8');
+              
+              const comments = await github.rest.issues.listComments({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number
+              });
+              
+              const existingComment = comments.data.find(
+                comment => comment.user.login === 'github-actions[bot]' &&
+                comment.body.includes('RAG System Evaluation Report')
+              );
+              
+              if (existingComment) {
+                await github.rest.issues.updateComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  comment_id: existingComment.id,
+                  body: reportContent
+                });
+              } else {
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  body: reportContent
+                });
+              }
+              
+            } catch (error) {
+              console.error('Failed to post test results:', error);
+              
+              await github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: `## RAG System Evaluation Report\n\n**Error generating test report**\n\nFailed to read or post test results. Check workflow logs for details.\n\nError: ${error.message}`
+              });
+            }
+            
+      - name: Check test results and fail if needed
+        if: always()
+        run: |
+           # Check if pytest ran (look at step output)
+           if [ "${{ steps.run_tests.outcome }}" == "failure" ]; then
+            echo "Tests ran but failed - this is expected if RAG performance is below threshold"
+           fi  
+           if [ -f "pytest_captured_results.json" ]; then
+            total_tests=$(jq '.total_tests // 0' pytest_captured_results.json)
+            passed_tests=$(jq '.passed_tests // 0' pytest_captured_results.json)
+
+            if [ "$total_tests" -eq 0 ]; then
+              echo "ERROR: No tests were executed"
+              exit 1
+            fi
+
+            pass_rate=$(awk "BEGIN {print ($passed_tests / $total_tests) * 100}")
+
+            echo "DeepEval Test Results:"
+            echo "Total Tests: $total_tests"
+            echo "Passed Tests: $passed_tests"
+            echo "Pass Rate: $pass_rate%"
+
+            if (( $(echo "$pass_rate < 70" | bc -l) )); then
+              echo "TEST FAILURE: Pass rate $pass_rate% is below threshold 70%"
+              echo "RAG system performance is below acceptable standards."
+              exit 1
+            else
+              echo "TEST SUCCESS: Pass rate $pass_rate% meets threshold 70%"
+            fi
+           else
+            echo "ERROR: No test results file found"
+            exit 1
+           fi