merge prod

Author: tarnopolskyi

uncoder-core/app/translator/core/context_vars.py

@@ -1,4 +1,4 @@
 from contextvars import ContextVar
 
 return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False)
-"""Set to True to return ony first query if rendered multiple options"""
+"""Set to True to return only first query if rendered multiple options"""

uncoder-core/app/translator/core/render.py

@@ -16,9 +16,10 @@
 limitations under the License.
 -----------------------------------------------------------------
 """
+
 from abc import ABC, abstractmethod
 from collections.abc import Callable
-from typing import Optional, Union
+from typing import ClassVar, Optional, Union
 
 from app.translator.const import DEFAULT_VALUE_TYPE
 from app.translator.core.context_vars import return_only_first_query_ctx_var
@@ -165,7 +166,14 @@ class QueryRender(ABC):
     is_single_line_comment: bool = False
     unsupported_functions_text = "Unsupported functions were excluded from the result query:"
 
-    platform_functions: PlatformFunctions = PlatformFunctions()
+    platform_functions: PlatformFunctions = None
+
+    def __init__(self):
+        self.init_platform_functions()
+
+    def init_platform_functions(self) -> None:
+        self.platform_functions = PlatformFunctions()
+        self.platform_functions.platform_query_render = self
 
     def render_not_supported_functions(self, not_supported_functions: list) -> str:
         line_template = f"{self.comment_symbol} " if self.comment_symbol and self.is_single_line_comment else ""
@@ -192,19 +200,19 @@ class PlatformQueryRender(QueryRender):
 
     field_value_map = BaseQueryFieldValue(or_token=or_token)
 
-    query_pattern = "{table} {query} {functions}"
-    raw_log_field_pattern_map: dict = None
+    raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
 
     def __init__(self):
+        super().__init__()
         self.operator_map = {
             LogicalOperatorType.AND: f" {self.and_token} ",
             LogicalOperatorType.OR: f" {self.or_token} ",
             LogicalOperatorType.NOT: f" {self.not_token} ",
         }
 
-    def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str:  # noqa: ARG002
-        if str(log_source_signature):
-            return f"{log_source_signature!s} {self.and_token}"
+    def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str:  # noqa: ARG002
+        if log_source_signature and str(log_source_signature):
+            return f"{log_source_signature} {self.and_token}"
         return ""
 
     def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions:
@@ -272,6 +280,10 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) ->
             query = f"{query}\n\n{query_meta_info}"
         return query
 
+    @staticmethod
+    def _finalize_search_query(query: str) -> str:
+        return query
+
     def finalize_query(
         self,
         prefix: str,
@@ -283,8 +295,8 @@ def finalize_query(
         *args,  # noqa: ARG002
         **kwargs,  # noqa: ARG002
     ) -> str:
-        query = self.query_pattern.format(prefix=prefix, query=query, functions=functions).strip()
-
+        parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions]))
+        query = " ".join(parts)
         query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query)
         if not_supported_functions:
             rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
@@ -327,15 +339,15 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer)
 
     def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
         if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type):
-            return raw_log_field_pattern.pattern.format(field=field)
+            return raw_log_field_pattern.format(field=field)
 
     def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]:
         if isinstance(field, list):
-            list_of_prefix = []
+            prefix_list = []
             for f in field:
-                if prepared_prefix := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping):
-                    list_of_prefix.extend(prepared_prefix)
-            return list_of_prefix
+                if _prefix_list := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping):
+                    prefix_list.extend(_prefix_list)
+            return prefix_list
         if raw_log_field_type := source_mapping.raw_log_fields.get(field):
             return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
 
@@ -352,9 +364,11 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
                 )
             if not mapped_field and self.is_strict_mapping:
                 raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
-            if field_prefix := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
-                defined_raw_log_fields.extend(field_prefix)
-        return "\n".join(set(defined_raw_log_fields))
+            if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
+                for prefix in prefix_list:
+                    if prefix not in defined_raw_log_fields:
+                        defined_raw_log_fields.append(prefix)
+        return "\n".join(defined_raw_log_fields)
 
     def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
         queries_map = {}

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml

@@ -77,6 +77,7 @@ field_mapping:
   OldTargetUserName: xdm.target.user.username
   UserPrincipalName: xdm.source.user.username
   DestAddress: xdm.target.ipv4
+  SubjectAccountName: xdm.source.user.username
   SubjectUserName: xdm.source.user.username
   SubjectUserSid: xdm.source.user.identifier
   SourceAddr: xdm.source.ipv4
@@ -117,7 +118,6 @@ field_mapping:
   method: xdm.network.http.method
   notice.user_agent: xdm.network.http.browser
   hasIdentity: xdm.source.user.identity_type
-  SubjectAccountName: xdm.source.user.username
   ComputerName: xdm.source.host.hostname
   ExternalSeverity: xdm.alert.severity
   SourceMAC: xdm.source.host.mac_addresses

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml

@@ -7,7 +7,8 @@ default_log_source:
 field_mapping:
   EventID: action_evtlog_event_id
   Provider_Name: provider_name
-
+  SubjectAccountName: actor_effective_username
+  
 raw_log_fields:
   ParentImage: regex
   AccessMask: regex

uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml

@@ -130,6 +130,9 @@ field_mapping:
   NewValue: NewValue
   Source: Source
   Status: Status
+  SubjectAccountName: 
+    - Subject Account Name
+    - SubjectAccountName
   SubjectDomainName: SubjectDomainName
   SubjectUserName: Target Username
   SubjectUserSid: SubjectUserSid
@@ -171,5 +174,4 @@ field_mapping:
   UserID: UserID
   ParentProcessName: Parent Process Name
   Service: Service
-  hasIdentity: hasIdentity
-  SubjectAccountName: SubjectAccountName
+  hasIdentity: hasIdentity

uncoder-core/app/translator/platforms/athena/renders/athena.py

@@ -16,6 +16,7 @@
 limitations under the License.
 -----------------------------------------------------------------
 """
+
 from app.translator.core.models.platform_details import PlatformDetails
 from app.translator.managers import render_manager
 from app.translator.platforms.athena.const import athena_details
@@ -35,6 +36,9 @@ class AthenaQueryRender(SqlQueryRender):
     or_token = "OR"
 
     field_value_map = AthenaFieldValue(or_token=or_token)
-    query_pattern = "{prefix} WHERE {query} {functions}"
     comment_symbol = "--"
     is_single_line_comment = True
+
+    @staticmethod
+    def _finalize_search_query(query: str) -> str:
+        return f"WHERE {query}" if query else ""

uncoder-core/app/translator/platforms/base/aql/renders/aql.py

@@ -16,6 +16,7 @@
 limitations under the License.
 -----------------------------------------------------------------
 """
+
 from typing import Union
 
 from app.translator.const import DEFAULT_VALUE_TYPE
@@ -127,7 +128,6 @@ class AQLQueryRender(PlatformQueryRender):
     not_token = "NOT"
 
     field_value_map = AQLFieldValue(or_token=or_token)
-    query_pattern = "{prefix} AND {query} {functions}"
 
     def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions_prefix: str = "") -> str:  # noqa: ARG002
         table = str(log_source_signature)
@@ -136,3 +136,7 @@ def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions
 
     def wrap_with_comment(self, value: str) -> str:
         return f"/* {value} */"
+
+    @staticmethod
+    def _finalize_search_query(query: str) -> str:
+        return f"AND {query}" if query else ""

uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py

@@ -16,6 +16,7 @@
 limitations under the License.
 -----------------------------------------------------------------
 """
+
 from typing import Optional, Union
 
 from app.translator.const import DEFAULT_VALUE_TYPE
@@ -106,8 +107,6 @@ class LuceneQueryRender(PlatformQueryRender):
     and_token = "AND"
     not_token = "NOT"
 
-    query_pattern = "{query} {functions}"
-
     comment_symbol = "//"
     is_single_line_comment = True
 

uncoder-core/app/translator/platforms/base/spl/renders/spl.py

@@ -16,6 +16,7 @@
 limitations under the License.
 -----------------------------------------------------------------
 """
+
 from typing import Union
 
 from app.translator.const import DEFAULT_VALUE_TYPE
@@ -78,7 +79,6 @@ class SplQueryRender(PlatformQueryRender):
     and_token = "AND"
     not_token = "NOT"
 
-    query_pattern = "{prefix} {query} {functions}"
     comment_symbol = "```"
 
     def wrap_with_comment(self, value: str) -> str:

uncoder-core/app/translator/platforms/base/sql/renders/sql.py

@@ -16,6 +16,7 @@
 limitations under the License.
 -----------------------------------------------------------------
 """
+
 from typing import Union
 
 from app.translator.const import DEFAULT_VALUE_TYPE
@@ -76,10 +77,13 @@ class SqlQueryRender(PlatformQueryRender):
     and_token = "AND"
     not_token = "NOT"
 
-    query_pattern = "{prefix} WHERE {query} {functions}"
     comment_symbol = "--"
     is_single_line_comment = True
 
     def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str:  # noqa: ARG002
         table = str(log_source_signature) if str(log_source_signature) else "eventlog"
         return f"SELECT * FROM {table}"
+
+    @staticmethod
+    def _finalize_search_query(query: str) -> str:
+        return f"WHERE {query}" if query else ""