Enhance language handling and context specificity in response generation

src/llm_orchestration_service.py

@@ -2664,7 +2664,7 @@ def _generate_rag_response(
                     llmServiceActive=False,
                     questionOutOfLLMScope=False,
                     inputGuardFailed=False,
-                    content=TECHNICAL_ISSUE_MESSAGE,
+                    content=localized_msg,
                 )
 
         try:
@@ -2821,7 +2821,7 @@ def _generate_rag_response(
                     llmServiceActive=False,
                     questionOutOfLLMScope=False,
                     inputGuardFailed=False,
-                    content=TECHNICAL_ISSUE_MESSAGE,
+                    content=localized_msg,
                 )
 
     # ========================================================================

src/optimization/optimized_module_loader.py

@@ -344,17 +344,28 @@ class ResponseGeneratorSignature(dspy.Signature):
             """
             Produce a grounded answer from the provided context ONLY.
 
+            CRITICAL LANGUAGE RULE:
+            - The answer MUST be in the SAME language as the input question
+            - Estonian question → Estonian answer
+            - Russian question → Russian answer
+            - English question → English answer
+            - Maintain the natural language flow and grammar of the detected language
+
             Rules:
-            - Use ONLY the provided context blocks
+            - Use ONLY the provided context blocks; do not invent facts
             - If context is insufficient, set questionOutOfLLMScope=true
-            - Do not invent facts
+            - Do not include citations in the answer field
             - Be concise and direct
             """
 
-            question: str = dspy.InputField(desc="User's question")
+            question: str = dspy.InputField(
+                desc="User's question. Answer in the SAME language as this question."
+            )
             context_blocks: list[str] = dspy.InputField(desc="Relevant context chunks")
 
-            answer: str = dspy.OutputField(desc="Grounded answer from context")
+            answer: str = dspy.OutputField(
+                desc="Human-friendly answer in THE SAME LANGUAGE as the question, without citations"
+            )
             questionOutOfLLMScope: bool = dspy.OutputField(
                 desc="True if context is insufficient to answer"
             )

src/optimization/optimized_modules/generator/generator_optimized_20260424_065431.json

@@ -0,0 +1,91 @@
+{
+  "predict": {
+    "traces": [],
+    "train": [],
+    "demos": [
+      {
+        "augmented": true,
+        "question": "Kui palju raha kulutati peretoetustele 2021. aastal kokku?",
+        "context_blocks": [
+          "[Context 1]\n2021. aastal maksti peredele toetusi kokku umbes 653 miljonit eurot. Umbes 310 miljonit eurot peretoetusteks; umbes 280 miljonit eurot vanemahüvitiseks; sünnitoetust umbes 58 miljonit eurot; ülalpidamistoetust umbes 5 miljonit eurot."
+        ],
+        "reasoning": "Kontekstist selgub, et 2021. aastal maksti peredele toetusi kokku umbes 653 miljonit eurot, mis sisaldab peretoetusi, vanemahüvitist, sünnitoetust ja ülalpidamistoetust.",
+        "answer": "2021. aastal kulutati peretoetustele kokku umbes 653 miljonit eurot.",
+        "questionOutOfLLMScope": false
+      },
+      {
+        "augmented": true,
+        "question": "Kes on töötutoetusele õigustatud?",
+        "context_blocks": [
+          "[Context 1]\nTöötutoetus on saadaval inimestele, kes on töötanud ja maksnud sotsiaalmaksu vähemalt 12 kuud viimase 36 kuu jooksul ning on registreeritud töötuna."
+        ],
+        "reasoning": "Kontekstist selgub, et töötutoetusele on õigus neil, kes on töötanud ja maksnud sotsiaalmaksu vähemalt 12 kuud viimase 36 kuu jooksul ning on registreeritud töötuna.",
+        "answer": "Töötutoetusele on õigustatud inimesed, kes on viimase 36 kuu jooksul vähemalt 12 kuud töötanud ja maksnud sotsiaalmaksu ning on registreeritud töötuna.",
+        "questionOutOfLLMScope": false
+      },
+      {
+        "augmented": true,
+        "question": "Milliseid filme näidatakse kinodes?",
+        "context_blocks": [
+          "[Context 1]\n(No relevant context available.)"
+        ],
+        "reasoning": "Kontekstis puudub teave selle kohta, milliseid filme praegu kinodes näidatakse.",
+        "answer": "Kahjuks puudub mul teave selle kohta, milliseid filme hetkel kinodes näidatakse.",
+        "questionOutOfLLMScope": true
+      },
+      {
+        "augmented": true,
+        "question": "What was the total amount spent on family benefits in 2021?",
+        "context_blocks": [
+          "[Context 1]\nIn 2021, a total of approximately 653 million euros in benefits were paid to families. Approximately 310 million euros for family benefits; approximately 280 million euros for parental benefit; Maternity benefit of approx. 58 million euros; Maintenance benefit of about 5 million euros."
+        ],
+        "reasoning": "The context specifies that in 2021, a total of approximately 653 million euros in benefits were paid to families. This total includes family benefits, parental benefit, maternity benefit, and maintenance benefit. Therefore, the total amount spent on family benefits in 2021 is 653 million euros.",
+        "answer": "In 2021, a total of approximately 653 million euros was spent on family benefits.",
+        "questionOutOfLLMScope": false
+      },
+      {
+        "augmented": true,
+        "question": "How does Estonia support single-parent families?",
+        "context_blocks": [
+          "[Context 1]\nSince January 2022, the Ministry of Social Affairs has been looking for solutions to support single-parent families. The support needs of single parents are much wider than just financial support. There is a chronic lack of time, lack of flexible childcare, and high expectation of better access to hobby education."
+        ],
+        "reasoning": "The context states that since January 2022, the Ministry of Social Affairs in Estonia has been seeking solutions to support single-parent families. It mentions that the needs of single parents go beyond financial support and include issues such as lack of time, insufficient flexible childcare, and a need for better access to hobby education. However, the context does not specify the exact forms of support currently provided, only that these areas are being considered.",
+        "answer": "Estonia recognizes that single-parent families need support beyond just financial help. Since January 2022, the Ministry of Social Affairs has been seeking solutions to address challenges such as lack of time, insufficient flexible childcare, and the need for better access to hobby education for single parents.",
+        "questionOutOfLLMScope": false
+      }
+    ],
+    "signature": {
+      "instructions": "Produce a grounded answer from the provided context ONLY.\n\nCRITICAL LANGUAGE RULE:\n- The answer MUST be in the SAME language as the input question\n- Estonian question → Estonian answer\n- Russian question → Russian answer\n- English question → English answer\n- Maintain the natural language flow and grammar of the detected language\n\nRules:\n- Use ONLY the provided context blocks; do not invent facts\n- If context is insufficient, set questionOutOfLLMScope=true\n- Do not include citations in the answer field\n- Be concise and direct",
+      "fields": [
+        {
+          "prefix": "Question:",
+          "description": "User's question. Answer in the SAME language as this question."
+        },
+        {
+          "prefix": "Context Blocks:",
+          "description": "Relevant context chunks"
+        },
+        {
+          "prefix": "Reasoning: Let's think step by step in order to",
+          "description": "${reasoning}"
+        },
+        {
+          "prefix": "Answer:",
+          "description": "Human-friendly answer in THE SAME LANGUAGE as the question, without citations"
+        },
+        {
+          "prefix": "Question Out Of LLM Scope:",
+          "description": "True if context is insufficient to answer"
+        }
+      ]
+    },
+    "lm": null
+  },
+  "metadata": {
+    "dependency_versions": {
+      "python": "3.12",
+      "dspy": "3.0.3",
+      "cloudpickle": "3.1"
+    }
+  }
+}

src/optimization/optimized_modules/generator/generator_optimized_20260424_065431_results.json

@@ -0,0 +1,23 @@
+{
+  "component": "generator",
+  "optimizer": "Bootstrap+MIPROv2",
+  "metric_type": "GeneratorMetric with DSPy SemanticF1",
+  "total_time_seconds": 1155.654662,
+  "phase_times": {
+    "bootstrap": 31.949774,
+    "mipro": 1123.703534
+  },
+  "trainset_size": 5,
+  "valset_size": 21,
+  "num_candidates": 20,
+  "validation_stats": {
+    "combined_score": 1.0,
+    "scope_accuracy": 1.0,
+    "in_scope_performance": 1.0,
+    "out_scope_performance": 1.0,
+    "total_examples": 21,
+    "in_scope_examples": 16,
+    "out_scope_examples": 5
+  },
+  "timestamp": "2026-04-24T08:26:51.305627"
+}

src/optimization/optimized_modules/guardrails/guardrails_optimized_20260424_065431.json

@@ -0,0 +1,91 @@
+{
+  "predict": {
+    "traces": [],
+    "train": [],
+    "demos": [
+      {
+        "augmented": true,
+        "question": "Kui palju raha kulutati peretoetustele 2021. aastal kokku?",
+        "context_blocks": [
+          "[Context 1]\n2021. aastal maksti peredele toetusi kokku umbes 653 miljonit eurot. Umbes 310 miljonit eurot peretoetusteks; umbes 280 miljonit eurot vanemahüvitiseks; sünnitoetust umbes 58 miljonit eurot; ülalpidamistoetust umbes 5 miljonit eurot."
+        ],
+        "reasoning": "Kontekstist selgub, et 2021. aastal maksti peredele toetusi kokku umbes 653 miljonit eurot, mis sisaldab peretoetusi, vanemahüvitist, sünnitoetust ja ülalpidamistoetust.",
+        "answer": "2021. aastal kulutati peretoetustele kokku umbes 653 miljonit eurot.",
+        "questionOutOfLLMScope": false
+      },
+      {
+        "augmented": true,
+        "question": "Kes on töötutoetusele õigustatud?",
+        "context_blocks": [
+          "[Context 1]\nTöötutoetus on saadaval inimestele, kes on töötanud ja maksnud sotsiaalmaksu vähemalt 12 kuud viimase 36 kuu jooksul ning on registreeritud töötuna."
+        ],
+        "reasoning": "Kontekstist selgub, et töötutoetusele on õigus neil, kes on töötanud ja maksnud sotsiaalmaksu vähemalt 12 kuud viimase 36 kuu jooksul ning on registreeritud töötuna.",
+        "answer": "Töötutoetusele on õigustatud inimesed, kes on viimase 36 kuu jooksul vähemalt 12 kuud töötanud ja maksnud sotsiaalmaksu ning on registreeritud töötuna.",
+        "questionOutOfLLMScope": false
+      },
+      {
+        "augmented": true,
+        "question": "Milliseid filme näidatakse kinodes?",
+        "context_blocks": [
+          "[Context 1]\n(No relevant context available.)"
+        ],
+        "reasoning": "Kontekstis puudub teave selle kohta, milliseid filme praegu kinodes näidatakse.",
+        "answer": "Kahjuks puudub mul teave selle kohta, milliseid filme hetkel kinodes näidatakse.",
+        "questionOutOfLLMScope": true
+      },
+      {
+        "augmented": true,
+        "question": "What was the total amount spent on family benefits in 2021?",
+        "context_blocks": [
+          "[Context 1]\nIn 2021, a total of approximately 653 million euros in benefits were paid to families. Approximately 310 million euros for family benefits; approximately 280 million euros for parental benefit; Maternity benefit of approx. 58 million euros; Maintenance benefit of about 5 million euros."
+        ],
+        "reasoning": "The context specifies that in 2021, a total of approximately 653 million euros in benefits were paid to families. This total includes family benefits, parental benefit, maternity benefit, and maintenance benefit. Therefore, the total amount spent on family benefits in 2021 is 653 million euros.",
+        "answer": "In 2021, a total of approximately 653 million euros was spent on family benefits.",
+        "questionOutOfLLMScope": false
+      },
+      {
+        "augmented": true,
+        "question": "How does Estonia support single-parent families?",
+        "context_blocks": [
+          "[Context 1]\nSince January 2022, the Ministry of Social Affairs has been looking for solutions to support single-parent families. The support needs of single parents are much wider than just financial support. There is a chronic lack of time, lack of flexible childcare, and high expectation of better access to hobby education."
+        ],
+        "reasoning": "The context states that since January 2022, the Ministry of Social Affairs in Estonia has been seeking solutions to support single-parent families. It mentions that the needs of single parents go beyond financial support and include issues such as lack of time, insufficient flexible childcare, and a need for better access to hobby education. However, the context does not specify the exact forms of support currently provided, only that these areas are being considered.",
+        "answer": "Estonia recognizes that single-parent families need support beyond just financial help. Since January 2022, the Ministry of Social Affairs has been seeking solutions to address challenges such as lack of time, insufficient flexible childcare, and the need for better access to hobby education for single parents.",
+        "questionOutOfLLMScope": false
+      }
+    ],
+    "signature": {
+      "instructions": "Produce a grounded answer from the provided context ONLY.\n\nCRITICAL LANGUAGE RULE:\n- The answer MUST be in the SAME language as the input question\n- Estonian question → Estonian answer\n- Russian question → Russian answer\n- English question → English answer\n- Maintain the natural language flow and grammar of the detected language\n\nRules:\n- Use ONLY the provided context blocks; do not invent facts\n- If context is insufficient, set questionOutOfLLMScope=true\n- Do not include citations in the answer field\n- Be concise and direct",
+      "fields": [
+        {
+          "prefix": "Question:",
+          "description": "User's question. Answer in the SAME language as this question."
+        },
+        {
+          "prefix": "Context Blocks:",
+          "description": "Relevant context chunks"
+        },
+        {
+          "prefix": "Reasoning: Let's think step by step in order to",
+          "description": "${reasoning}"
+        },
+        {
+          "prefix": "Answer:",
+          "description": "Human-friendly answer in THE SAME LANGUAGE as the question, without citations"
+        },
+        {
+          "prefix": "Question Out Of LLM Scope:",
+          "description": "True if context is insufficient to answer"
+        }
+      ]
+    },
+    "lm": null
+  },
+  "metadata": {
+    "dependency_versions": {
+      "python": "3.12",
+      "dspy": "3.0.3",
+      "cloudpickle": "3.1"
+    }
+  }
+}

src/optimization/optimized_modules/guardrails/guardrails_optimized_20260424_065431_results.json

@@ -0,0 +1,27 @@
+{
+  "component": "guardrails",
+  "optimizer": "MIPROv2-light",
+  "optimization_time_seconds": 1411.58703,
+  "trainset_size": 10,
+  "valset_size": 44,
+  "num_candidates": 10,
+  "validation_stats": {
+    "raw_accuracy": 0.9545454545454546,
+    "weighted_accuracy": 0.9545454545454546,
+    "precision": 1.0,
+    "recall": 0.9130434782608695,
+    "f1_score": 0.9545454545454545,
+    "confusion_matrix": {
+      "tp": 21,
+      "tn": 21,
+      "fp": 0,
+      "fn": 2
+    },
+    "true_positives": 21,
+    "true_negatives": 21,
+    "false_positives": 0,
+    "false_negatives": 2,
+    "total_predictions": 44
+  },
+  "timestamp": "2026-04-24T07:22:35.530732"
+}

src/optimization/optimized_modules/refiner/refiner_optimized_20260424_065431.json

@@ -0,0 +1,98 @@
+{
+  "predict": {
+    "traces": [],
+    "train": [],
+    "demos": [
+      {
+        "augmented": true,
+        "history": "",
+        "question": "poverty stats",
+        "n": 3,
+        "reasoning": "The user asked a very broad question: \"poverty stats\". There is no prior context, so the rewrites should clarify and specify the request, making it more explicit and searchable. Since no location, time period, or demographic is given, the rewrites should cover common interpretations, such as national poverty statistics, recent data, and possibly global comparisons. Each rewrite should be distinct, focusing on different ways someone might seek poverty-related statistical information.",
+        "rewrites": [
+          "What are the latest poverty statistics for Estonia?",
+          "Can you provide recent global poverty rates?",
+          "What percentage of the population in Estonia lives below the poverty line?"
+        ]
+      },
+      {
+        "history": "",
+        "question": "large families",
+        "n": 3,
+        "expected_refinements": [
+          "What benefits are available for large families in Estonia?",
+          "How does Estonia support families with many children?",
+          "What is the definition of a large family for benefit purposes in Estonia?"
+        ],
+        "rewrites": [
+          "What benefits are available for large families in Estonia?",
+          "How does Estonia support families with many children?",
+          "What is the definition of a large family for benefit purposes in Estonia?"
+        ]
+      },
+      {
+        "history": "user: Vajan infot töötuse kohta\nassistant: Eesti pakub töötutoetust ja tööhõiveteenuseid Töötukassa kaudu.",
+        "question": "Kui kaua?",
+        "n": 3,
+        "expected_refinements": [
+          "Kui kaua ma saan töötutoetust Eestis?",
+          "Mis on töötuskindlustuse kestus Eestis?",
+          "Mitu kuud kestab töötutoetus Eestis?"
+        ],
+        "rewrites": [
+          "Kui kaua ma saan töötutoetust Eestis?",
+          "Mis on töötuskindlustuse kestus Eestis?",
+          "Mitu kuud kestab töötutoetus Eestis?"
+        ]
+      },
+      {
+        "history": "user: What's the parental benefit like?\nassistant: Estonia's parental benefit is one of the most generous globally, covering a long period with substantial amounts.",
+        "question": "Compared to what?",
+        "n": 3,
+        "expected_refinements": [
+          "How does Estonia's parental benefit compare to other EU countries?",
+          "What is Estonia's ranking for parental benefits in the EU?",
+          "How does Estonian parental benefit compare to the European average?"
+        ],
+        "rewrites": [
+          "How does Estonia's parental benefit compare to other EU countries?",
+          "What is Estonia's ranking for parental benefits in the EU?",
+          "How does Estonian parental benefit compare to the European average?"
+        ]
+      }
+    ],
+    "signature": {
+      "instructions": "Produce N distinct, concise rewrites of the user's question using chat history.\n\nConstraints:\n- Preserve the original intent\n- Resolve pronouns with context when safe\n- Prefer explicit, searchable phrasing (entities, dates, units)\n- Make each rewrite meaningfully distinct\n- Return exactly N items as a list",
+      "fields": [
+        {
+          "prefix": "History:",
+          "description": "Recent conversation history"
+        },
+        {
+          "prefix": "Question:",
+          "description": "The user's latest question to refine"
+        },
+        {
+          "prefix": "N:",
+          "description": "Number of rewrites to produce"
+        },
+        {
+          "prefix": "Reasoning: Let's think step by step in order to",
+          "description": "${reasoning}"
+        },
+        {
+          "prefix": "Rewrites:",
+          "description": "Exactly N refined variations of the question"
+        }
+      ]
+    },
+    "lm": null
+  },
+  "metadata": {
+    "dependency_versions": {
+      "python": "3.12",
+      "dspy": "3.0.3",
+      "cloudpickle": "3.1"
+    }
+  }
+}