merge prod

Author: tarnopolskyi

uncoder-core/app/translator/core/mapping.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Optional, TypeVar
+from typing import TYPE_CHECKING, Optional, TypeVar, Union
 
 from app.translator.core.exceptions.core import StrictPlatformException
 from app.translator.core.models.platform_details import PlatformDetails
@@ -19,9 +19,14 @@ class LogSourceSignature(ABC):
     wildcard_symbol = "*"
 
     @abstractmethod
-    def is_suitable(self, *args, **kwargs) -> bool:
+    def is_suitable(self, **kwargs) -> bool:
         raise NotImplementedError("Abstract method")
 
+    @staticmethod
+    def _check_conditions(conditions: list[Union[bool, None]]) -> bool:
+        conditions = [condition for condition in conditions if condition is not None]
+        return bool(conditions) and all(conditions)
+
     @abstractmethod
     def __str__(self) -> str:
         raise NotImplementedError("Abstract method")
@@ -70,7 +75,7 @@ def update(self, fields_mapping: FieldsMapping) -> None:
         self.__render_mapping.update(fields_mapping.__render_mapping)
 
     def is_suitable(self, field_names: list[str]) -> bool:
-        return set(field_names).issubset(set(self.__parser_mapping.keys()))
+        return bool(field_names) and set(field_names).issubset(set(self.__parser_mapping.keys()))
 
 
 _LogSourceSignatureType = TypeVar("_LogSourceSignatureType", bound=LogSourceSignature)
@@ -147,9 +152,23 @@ def prepare_fields_mapping(field_mapping: dict) -> FieldsMapping:
     def prepare_log_source_signature(self, mapping: dict) -> LogSourceSignature:
         raise NotImplementedError("Abstract method")
 
-    @abstractmethod
-    def get_suitable_source_mappings(self, *args, **kwargs) -> list[SourceMapping]:
-        raise NotImplementedError("Abstract method")
+    def get_suitable_source_mappings(
+        self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]]
+    ) -> list[SourceMapping]:
+        by_log_sources_and_fields = []
+        by_fields = []
+        for source_mapping in self._source_mappings.values():
+            if source_mapping.source_id == DEFAULT_MAPPING_NAME:
+                continue
+
+            if source_mapping.fields_mapping.is_suitable(field_names):
+                by_fields.append(source_mapping)
+
+                log_source_signature: LogSourceSignature = source_mapping.log_source_signature
+                if log_source_signature and log_source_signature.is_suitable(**log_sources):
+                    by_log_sources_and_fields.append(source_mapping)
+
+        return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]]
 
     def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
         return self._source_mappings.get(source_id)

uncoder-core/app/translator/core/models/query_container.py

@@ -77,6 +77,14 @@ def __init__(
     def author_str(self) -> str:
         return ", ".join(self.author)
 
+    @property
+    def source_mapping_ids(self) -> list[str]:
+        return sorted(self._source_mapping_ids)
+
+    @source_mapping_ids.setter
+    def source_mapping_ids(self, source_mapping_ids: list[str]) -> None:
+        self._source_mapping_ids = source_mapping_ids
+
 
 @dataclass
 class RawQueryContainer:

uncoder-core/app/translator/core/models/query_tokens/field.py

@@ -1,3 +1,4 @@
+from abc import ABC, abstractmethod
 from typing import Optional
 
 from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping
@@ -37,3 +38,10 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
 class PredefinedField:
     def __init__(self, name: str):
         self.name = name
+
+
+class BaseFieldsGetter(ABC):
+    @property
+    @abstractmethod
+    def fields(self) -> list[Field]:
+        raise NotImplementedError("Abstract method")

uncoder-core/app/translator/core/models/query_tokens/field_field.py

@@ -1,8 +1,8 @@
-from app.translator.core.models.query_tokens.field import Alias, Field
+from app.translator.core.models.query_tokens.field import Alias, BaseFieldsGetter, Field
 from app.translator.core.models.query_tokens.identifier import Identifier
 
 
-class FieldField:
+class FieldField(BaseFieldsGetter):
     def __init__(
         self,
         source_name_left: str,
@@ -16,3 +16,13 @@ def __init__(
         self.operator = operator
         self.field_right = Field(source_name=source_name_right) if not is_alias_right else None
         self.alias_right = Alias(name=source_name_right) if is_alias_right else None
+
+    @property
+    def fields(self) -> list[Field]:
+        fields = []
+        if self.field_left:
+            fields.append(self.field_left)
+        if self.field_right:
+            fields.append(self.field_right)
+
+        return fields

uncoder-core/app/translator/core/models/query_tokens/field_value.py

@@ -1,13 +1,13 @@
 from typing import Union
 
 from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS
-from app.translator.core.models.query_tokens.field import Alias, Field, PredefinedField
+from app.translator.core.models.query_tokens.field import Alias, BaseFieldsGetter, Field, PredefinedField
 from app.translator.core.models.query_tokens.identifier import Identifier
 from app.translator.core.models.query_tokens.value import Value
 from app.translator.core.str_value_manager import StrValue
 
 
-class FieldValue(Value):
+class FieldValue(BaseFieldsGetter, Value):
     def __init__(
         self,
         source_name: str,
@@ -33,3 +33,7 @@ def __repr__(self):
             return f"{self.predefined_field.name} {self.operator.token_type} {self.values}"
 
         return f"{self.field.source_name} {self.operator.token_type} {self.values}"
+
+    @property
+    def fields(self) -> list[Field]:
+        return [self.field] if self.field else []

uncoder-core/app/translator/core/models/query_tokens/function_value.py

@@ -2,13 +2,18 @@
 
 from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS
 from app.translator.core.models.functions.base import Function
+from app.translator.core.models.query_tokens.field import BaseFieldsGetter, Field
 from app.translator.core.models.query_tokens.identifier import Identifier
 from app.translator.core.models.query_tokens.value import Value
 from app.translator.core.str_value_manager import StrValue
 
 
-class FunctionValue(Value):
+class FunctionValue(BaseFieldsGetter, Value):
     def __init__(self, function: Function, operator: Identifier, value: Union[int, str, StrValue, list, tuple]):
         super().__init__(value, cast_to_int=operator.token_type not in STR_SEARCH_OPERATORS)
         self.function = function
         self.operator = operator
+
+    @property
+    def fields(self) -> list[Field]:
+        return self.function.fields

uncoder-core/app/translator/core/parser.py

@@ -62,30 +62,24 @@ def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]:
             raise TokenizerGeneralException("Can't translate empty query. Please provide more details")
         return self.tokenizer.tokenize(query=query)
 
+    @staticmethod
     def get_field_tokens(
-        self, query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
+        query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
     ) -> list[Field]:
         field_tokens = []
         for token in query_tokens:
-            if isinstance(token, FieldValue):
-                field_tokens.append(token.field)
-            elif isinstance(token, FieldField):
-                if token.field_left:
-                    field_tokens.append(token.field_left)
-                if token.field_right:
-                    field_tokens.append(token.field_right)
-            elif isinstance(token, FunctionValue):
-                field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args([token.function]))
+            if isinstance(token, (FieldField, FieldValue, FunctionValue)):
+                field_tokens.extend(token.fields)
 
         if functions:
-            field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args(functions))
+            field_tokens.extend([field for func in functions for field in func.fields])
 
         return field_tokens
 
     def get_source_mappings(
-        self, field_tokens: list[Field], log_sources: dict[str, Union[str, list[str]]]
+        self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]]
     ) -> list[SourceMapping]:
         field_names = [field.source_name for field in field_tokens]
-        source_mappings = self.mappings.get_suitable_source_mappings(field_names=field_names, **log_sources)
+        source_mappings = self.mappings.get_suitable_source_mappings(field_names=field_names, log_sources=log_sources)
         self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping)
         return source_mappings

uncoder-core/app/translator/core/str_value_manager.py

@@ -42,6 +42,10 @@ class ReEndOfStrSymbol(BaseSpecSymbol):
     ...
 
 
+class ReWordBoundarySymbol(BaseSpecSymbol):
+    ...
+
+
 class ReWordSymbol(BaseSpecSymbol):
     ...
 
@@ -130,6 +134,7 @@ def has_spec_symbols(self) -> bool:
     SingleSymbolWildCard: "?",
     UnboundLenWildCard: "*",
     ReAnySymbol: ".",
+    ReWordBoundarySymbol: r"\b",
     ReWordSymbol: r"\w",
     ReDigitalSymbol: r"\d",
     ReWhiteSpaceSymbol: r"\s",

uncoder-core/app/translator/core/tokenizer.py

@@ -31,13 +31,6 @@
 )
 from app.translator.core.functions import PlatformFunctions
 from app.translator.core.mapping import SourceMapping
-from app.translator.core.models.functions.base import Function
-from app.translator.core.models.functions.eval import EvalArg
-from app.translator.core.models.functions.group_by import GroupByFunction
-from app.translator.core.models.functions.join import JoinFunction
-from app.translator.core.models.functions.rename import RenameArg
-from app.translator.core.models.functions.sort import SortArg
-from app.translator.core.models.functions.union import UnionFunction
 from app.translator.core.models.query_tokens.field import Field
 from app.translator.core.models.query_tokens.field_field import FieldField
 from app.translator.core.models.query_tokens.field_value import FieldValue
@@ -356,43 +349,6 @@ def filter_tokens(
     ) -> list[QUERY_TOKEN_TYPE]:
         return [token for token in tokens if isinstance(token, token_type)]
 
-    def get_field_tokens_from_func_args(  # noqa: PLR0912
-        self, args: list[Union[Field, FieldValue, Keyword, Identifier, Function, SortArg]]
-    ) -> list[Field]:
-        result = []
-        for arg in args:
-            if isinstance(arg, Field):
-                result.append(arg)
-            elif isinstance(arg, FieldField):
-                if arg.field_left:
-                    result.append(arg.field_left)
-                if arg.field_right:
-                    result.append(arg.field_right)
-            elif isinstance(arg, FieldValue):
-                if arg.field:
-                    result.append(arg.field)
-            elif isinstance(arg, FunctionValue):
-                result.extend(self.get_field_tokens_from_func_args(args=[arg.function]))
-            elif isinstance(arg, GroupByFunction):
-                result.extend(self.get_field_tokens_from_func_args(args=arg.args))
-                result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))
-                result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_]))
-            elif isinstance(arg, JoinFunction):
-                result.extend(self.get_field_tokens_from_func_args(args=arg.condition))
-            elif isinstance(arg, UnionFunction):
-                continue
-            elif isinstance(arg, Function):
-                result.extend(self.get_field_tokens_from_func_args(args=arg.args))
-            elif isinstance(arg, SortArg) and isinstance(arg.field, Field):
-                result.append(arg.field)
-            elif isinstance(arg, RenameArg):
-                result.append(arg.field_)
-            elif isinstance(arg, EvalArg):
-                if isinstance(arg.field_, Field):
-                    result.append(arg.field_)
-                result.extend(self.get_field_tokens_from_func_args(args=arg.expression))
-        return result
-
     @staticmethod
     def set_field_tokens_generic_names_map(
         tokens: list[Field], source_mappings: list[SourceMapping], default_mapping: SourceMapping

uncoder-core/app/translator/mappings/platforms/qradar/default.yml

@@ -14,8 +14,6 @@ field_mapping:
     - DstPort
     - DestinationPort
     - remoteport
-  dst-hostname: DstHost
-  src-hostname: SrcHost
   src-port:
     - SourcePort
     - localport
@@ -25,21 +23,23 @@ field_mapping:
     - source_ip
     - SourceIP
     - sourceIP
+    - SrcHost
   dst-ip:
     - DestinationIP
     - destinationip
     - destination_ip
     - destinationIP
     - destinationaddress
     - destination
+    - DstHost
   User:
     - userName
     - EventUserName
     - Alert Threat Cause Actor Name
     - Username
     - Security ID
   CommandLine: Command
-  Protocol: 
+  Protocol:
     - IPProtocol
     - protocol
   Application:
@@ -97,8 +97,10 @@ field_mapping:
   FileName: 
     - Filename
     - File Name
-  RegistryKey: 
+    - Encoded Filename
+  RegistryKey:
     - Registry Key
     - Target Object
   RegistryValue: RegistryValue
   ProcessPath: Process Path
+  hasIdentity: hasIdentity