predefined field class

Author: alexvolha

uncoder-core/app/translator/core/custom_types/predefined_fields.py

@@ -0,0 +1,12 @@
+from app.translator.tools.custom_enum import CustomEnum
+
+
+class IPLocationType(CustomEnum):
+    asn = "ip_loc_asn"
+    asn_org = "ip_loc_asn_org"
+    city = "ip_loc_city"
+    continent = "ip_loc_continent"
+    country = "ip_loc_country"
+    lat_lon = "ip_loc_lat_lon"
+    region = "ip_loc_region"
+    timezone = "ip_loc_timezone"

uncoder-core/app/translator/core/models/field.py

@@ -37,6 +37,11 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
         self.__generic_names_map = generic_names_map
 
 
+class PredefinedField:
+    def __init__(self, name: str):
+        self.name = name
+
+
 class FieldField:
     def __init__(
         self,
@@ -46,10 +51,10 @@ def __init__(
         is_alias_left: bool = False,
         is_alias_right: bool = False,
     ):
-        self.field_left = Field(source_name=source_name_left)
+        self.field_left = Field(source_name=source_name_left) if not is_alias_left else None
         self.alias_left = Alias(name=source_name_left) if is_alias_left else None
         self.operator = operator
-        self.field_right = Field(source_name=source_name_right)
+        self.field_right = Field(source_name=source_name_right) if not is_alias_right else None
         self.alias_right = Alias(name=source_name_right) if is_alias_right else None
 
 
@@ -60,11 +65,14 @@ def __init__(
         operator: Identifier,
         value: Union[int, str, StrValue, list, tuple],
         is_alias: bool = False,
+        is_predefined_field: bool = False,
     ):
-        self.field = Field(source_name=source_name)
-        self.alias = None
-        if is_alias:
-            self.alias = Alias(name=source_name)
+        # mapped by platform fields mapping
+        self.field = Field(source_name=source_name) if not (is_alias or is_predefined_field) else None
+        # not mapped
+        self.alias = Alias(name=source_name) if is_alias else None
+        # mapped by platform predefined fields mapping
+        self.predefined_field = PredefinedField(name=source_name) if is_predefined_field else None
 
         self.operator = operator
         self.values = []
@@ -96,10 +104,13 @@ def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -
             self.values.append(value)
 
     def __repr__(self):
-        if self.field:
-            return f"{self.field.source_name} {self.operator.token_type} {self.values}"
+        if self.alias:
+            return f"{self.alias.name} {self.operator.token_type} {self.values}"
+
+        if self.predefined_field:
+            return f"{self.predefined_field.name} {self.operator.token_type} {self.values}"
 
-        return f"{self.alias.name} {self.operator.token_type} {self.values}"
+        return f"{self.field.source_name} {self.operator.token_type} {self.values}"
 
 
 class Keyword:

uncoder-core/app/translator/core/render.py

@@ -31,7 +31,7 @@
 from app.translator.core.exceptions.parser import UnsupportedOperatorException
 from app.translator.core.functions import PlatformFunctions
 from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
-from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
+from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField
 from app.translator.core.models.functions.base import Function, RenderedFunctions
 from app.translator.core.models.identifier import Identifier
 from app.translator.core.models.platform_details import PlatformDetails
@@ -218,7 +218,8 @@ class PlatformQueryRender(QueryRender):
     field_field_render = BaseFieldFieldRender()
     field_value_render = BaseFieldValueRender(or_token=or_token)
 
-    raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
+    predefined_fields_map: ClassVar[dict[str, str]] = {}
+    raw_log_field_patterns_map: ClassVar[dict[str, str]] = {}
 
     def __init__(self):
         super().__init__()
@@ -248,9 +249,23 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
 
         return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]
 
+    def map_predefined_field(self, predefined_field: PredefinedField) -> str:
+        if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
+            if self.is_strict_mapping:
+                raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name)
+
+            return predefined_field.name
+
+        return mapped_predefined_field_name
+
     def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
         if isinstance(token, FieldValue):
-            mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping)
+            if token.alias:
+                mapped_fields = [token.alias.name]
+            elif token.predefined_field:
+                mapped_fields = [self.map_predefined_field(token.predefined_field)]
+            else:
+                mapped_fields = self.map_field(token.field, source_mapping)
             joined = self.logical_operators_map[LogicalOperatorType.OR].join(
                 [
                     self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value)
@@ -365,7 +380,7 @@ def generate_from_raw_query_container(self, query_container: RawQueryContainer)
         )
 
     def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
-        if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type):
+        if raw_log_field_pattern := self.raw_log_field_patterns_map.get(field_type):
             return raw_log_field_pattern.format(field=field)
 
     def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]:
@@ -379,7 +394,7 @@ def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping
             return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
 
     def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str:
-        if self.raw_log_field_pattern_map is None:
+        if not self.raw_log_field_patterns_map:
             return ""
         defined_raw_log_fields = []
         for field in fields:

uncoder-core/app/translator/core/tokenizer.py

@@ -332,12 +332,12 @@ def get_field_tokens_from_func_args(  # noqa: PLR0912
             if isinstance(arg, Field):
                 result.append(arg)
             elif isinstance(arg, FieldField):
-                if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name:
+                if arg.field_left:
                     result.append(arg.field_left)
-                if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name:
+                if arg.field_right:
                     result.append(arg.field_right)
             elif isinstance(arg, FieldValue):
-                if not arg.alias or arg.alias.name != arg.field.source_name:
+                if arg.field:
                     result.append(arg.field)
             elif isinstance(arg, GroupByFunction):
                 result.extend(self.get_field_tokens_from_func_args(args=arg.args))

uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py

@@ -219,7 +219,7 @@ def generate_prefix(self, log_source_signature: LogSourceSignature, functions_pr
         return str(log_source_signature)
 
     def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
-        if isinstance(token, FieldValue):
+        if isinstance(token, FieldValue) and token.field:
             try:
                 mapped_fields = self.map_field(token.field, source_mapping)
             except StrictPlatformException:

uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py

@@ -79,7 +79,7 @@ def finalize_query(
         return self.wrap_with_not_supported_functions(rule_str, not_supported_functions)
 
     def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
-        if isinstance(token, FieldValue):
+        if isinstance(token, FieldValue) and token.field:
             for field in self.map_field(token.field, source_mapping):
                 self.fields.update({field: f"{{ctx.results.0.hits.hits.0._source.{field}}}"})
         return super().apply_token(token, source_mapping)

uncoder-core/app/translator/platforms/palo_alto/const.py

@@ -1,3 +1,4 @@
+from app.translator.core.custom_types.predefined_fields import IPLocationType
 from app.translator.core.models.platform_details import PlatformDetails
 
 PLATFORM_DETAILS = {"group_id": "cortex", "group_name": "Palo Alto Cortex XSIAM"}
@@ -10,3 +11,15 @@
 }
 
 cortex_xql_query_details = PlatformDetails(**CORTEX_XSIAM_XQL_QUERY_DETAILS)
+
+
+PREDEFINED_FIELDS_MAP = {
+    IPLocationType.asn: "loc_asn",
+    IPLocationType.asn_org: "loc_asn_org",
+    IPLocationType.city: "loc_city",
+    IPLocationType.continent: "loc_continent",
+    IPLocationType.country: "loc_country",
+    IPLocationType.lat_lon: "loc_latlon",
+    IPLocationType.region: "loc_region",
+    IPLocationType.timezone: "loc_timezone",
+}

uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py

@@ -30,7 +30,7 @@
 from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender
 from app.translator.core.str_value_manager import StrValue
 from app.translator.managers import render_manager
-from app.translator.platforms.palo_alto.const import cortex_xql_query_details
+from app.translator.platforms.palo_alto.const import PREDEFINED_FIELDS_MAP, cortex_xql_query_details
 from app.translator.platforms.palo_alto.functions import CortexXQLFunctions, cortex_xql_functions
 from app.translator.platforms.palo_alto.mapping import (
     CortexXQLLogSourceSignature,
@@ -167,7 +167,8 @@ class CortexXQLQueryRender(PlatformQueryRender):
     details: PlatformDetails = cortex_xql_query_details
     mappings: CortexXQLMappings = cortex_xql_mappings
     is_strict_mapping = True
-    raw_log_field_pattern_map: ClassVar[dict[str, str]] = {
+    predefined_fields_map = PREDEFINED_FIELDS_MAP
+    raw_log_field_patterns_map: ClassVar[dict[str, str]] = {
         "regex": '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")',
         "object": '| alter {field_name} = json_extract_scalar({field_object} , "$.{field_path}")',
         "list": '| alter {field_name} = arraystring(json_extract_array({field_object} , "$.{field_path}")," ")',
@@ -189,7 +190,7 @@ def init_platform_functions(self) -> None:
         self.platform_functions.platform_query_render = self
 
     def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
-        raw_log_field_pattern = self.raw_log_field_pattern_map.get(field_type)
+        raw_log_field_pattern = self.raw_log_field_patterns_map.get(field_type)
         if raw_log_field_pattern is None:
             return
         if field_type == "regex":
@@ -206,7 +207,7 @@ def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, fun
         return f"{functions_prefix}{log_source_str}"
 
     def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
-        if isinstance(token, FieldValue):
+        if isinstance(token, FieldValue) and token.field:
             field_name = token.field.source_name
             if values_map := SOURCE_MAPPING_TO_FIELD_VALUE_MAP.get(source_mapping.source_id, {}).get(field_name):
                 values_to_update = []