Merge pull request #166 from UncoderIO/gis-8036

predefined field processing

Author: alexvolha

uncoder-core/app/translator/core/custom_types/functions.py

@@ -15,6 +15,7 @@ class FunctionType(CustomEnum):
     latest = "latest"
 
     divide = "divide"
+    multiply = "multiply"
 
     lower = "lower"
     split = "split"
@@ -28,6 +29,7 @@ class FunctionType(CustomEnum):
     bin = "bin"
     eval = "eval"
     fields = "fields"
+    iploc = "iploc"
     join = "join"
     rename = "rename"
     search = "search"

uncoder-core/app/translator/core/custom_types/predefined_fields.py

@@ -0,0 +1,12 @@
+from app.translator.tools.custom_enum import CustomEnum
+
+
+class IPLocationType(CustomEnum):
+    asn = "ip_loc_asn"
+    asn_org = "ip_loc_asn_org"
+    city = "ip_loc_city"
+    continent = "ip_loc_continent"
+    country = "ip_loc_country"
+    lat_lon = "ip_loc_lat_lon"
+    region = "ip_loc_region"
+    timezone = "ip_loc_timezone"

uncoder-core/app/translator/core/models/field.py

@@ -37,6 +37,11 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
         self.__generic_names_map = generic_names_map
 
 
+class PredefinedField:
+    def __init__(self, name: str):
+        self.name = name
+
+
 class FieldField:
     def __init__(
         self,
@@ -46,10 +51,10 @@ def __init__(
         is_alias_left: bool = False,
         is_alias_right: bool = False,
     ):
-        self.field_left = Field(source_name=source_name_left)
+        self.field_left = Field(source_name=source_name_left) if not is_alias_left else None
         self.alias_left = Alias(name=source_name_left) if is_alias_left else None
         self.operator = operator
-        self.field_right = Field(source_name=source_name_right)
+        self.field_right = Field(source_name=source_name_right) if not is_alias_right else None
         self.alias_right = Alias(name=source_name_right) if is_alias_right else None
 
 
@@ -60,11 +65,14 @@ def __init__(
         operator: Identifier,
         value: Union[int, str, StrValue, list, tuple],
         is_alias: bool = False,
+        is_predefined_field: bool = False,
     ):
-        self.field = Field(source_name=source_name)
-        self.alias = None
-        if is_alias:
-            self.alias = Alias(name=source_name)
+        # mapped by platform fields mapping
+        self.field = Field(source_name=source_name) if not (is_alias or is_predefined_field) else None
+        # not mapped
+        self.alias = Alias(name=source_name) if is_alias else None
+        # mapped by platform predefined fields mapping
+        self.predefined_field = PredefinedField(name=source_name) if is_predefined_field else None
 
         self.operator = operator
         self.values = []
@@ -96,10 +104,13 @@ def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -
             self.values.append(value)
 
     def __repr__(self):
-        if self.field:
-            return f"{self.field.source_name} {self.operator.token_type} {self.values}"
+        if self.alias:
+            return f"{self.alias.name} {self.operator.token_type} {self.values}"
+
+        if self.predefined_field:
+            return f"{self.predefined_field.name} {self.operator.token_type} {self.values}"
 
-        return f"{self.alias.name} {self.operator.token_type} {self.values}"
+        return f"{self.field.source_name} {self.operator.token_type} {self.values}"
 
 
 class Keyword:

uncoder-core/app/translator/core/render.py

@@ -31,7 +31,7 @@
 from app.translator.core.exceptions.parser import UnsupportedOperatorException
 from app.translator.core.functions import PlatformFunctions
 from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
-from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
+from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField
 from app.translator.core.models.functions.base import Function, RenderedFunctions
 from app.translator.core.models.identifier import Identifier
 from app.translator.core.models.platform_details import PlatformDetails
@@ -218,7 +218,8 @@ class PlatformQueryRender(QueryRender):
     field_field_render = BaseFieldFieldRender()
     field_value_render = BaseFieldValueRender(or_token=or_token)
 
-    raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
+    predefined_fields_map: ClassVar[dict[str, str]] = {}
+    raw_log_field_patterns_map: ClassVar[dict[str, str]] = {}
 
     def __init__(self):
         super().__init__()
@@ -248,9 +249,23 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
 
         return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]
 
+    def map_predefined_field(self, predefined_field: PredefinedField) -> str:
+        if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
+            if self.is_strict_mapping:
+                raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name)
+
+            return predefined_field.name
+
+        return mapped_predefined_field_name
+
     def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
         if isinstance(token, FieldValue):
-            mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping)
+            if token.alias:
+                mapped_fields = [token.alias.name]
+            elif token.predefined_field:
+                mapped_fields = [self.map_predefined_field(token.predefined_field)]
+            else:
+                mapped_fields = self.map_field(token.field, source_mapping)
             joined = self.logical_operators_map[LogicalOperatorType.OR].join(
                 [
                     self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value)
@@ -365,7 +380,7 @@ def generate_from_raw_query_container(self, query_container: RawQueryContainer)
         )
 
     def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
-        if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type):
+        if raw_log_field_pattern := self.raw_log_field_patterns_map.get(field_type):
             return raw_log_field_pattern.format(field=field)
 
     def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]:
@@ -379,7 +394,7 @@ def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping
             return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
 
     def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str:
-        if self.raw_log_field_pattern_map is None:
+        if not self.raw_log_field_patterns_map:
             return ""
         defined_raw_log_fields = []
         for field in fields:

uncoder-core/app/translator/core/tokenizer.py

@@ -332,12 +332,12 @@ def get_field_tokens_from_func_args(  # noqa: PLR0912
             if isinstance(arg, Field):
                 result.append(arg)
             elif isinstance(arg, FieldField):
-                if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name:
+                if arg.field_left:
                     result.append(arg.field_left)
-                if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name:
+                if arg.field_right:
                     result.append(arg.field_right)
             elif isinstance(arg, FieldValue):
-                if not arg.alias or arg.alias.name != arg.field.source_name:
+                if arg.field:
                     result.append(arg.field)
             elif isinstance(arg, GroupByFunction):
                 result.extend(self.get_field_tokens_from_func_args(args=arg.args))

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml

@@ -11,4 +11,4 @@ field_mapping:
   dns_query_name: xdm.network.dns.dns_question.name
   QueryName: xdm.network.dns.dns_question.name
   query: xdm.network.dns.dns_question.name
-  dns-record-type: xdm.network.dns.dns_question.type
+  dns-record-type: xdm.network.dns.dns_question.type

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/linux_process_creation.yml

@@ -26,4 +26,5 @@ field_mapping:
   ParentProduct: actor_process_signature_product
   ParentCompany: actor_process_signature_vendor
   md5: action_process_image_md5
-  sha256: action_process_image_sha256
+  sha256: action_process_image_sha256
+  EventID: action_evtlog_event_id

uncoder-core/app/translator/mappings/platforms/qradar/default.yml

@@ -59,7 +59,9 @@ field_mapping:
     - dst-packets
   src-bytes: src-bytes
   dst-bytes: dst-bytes
-  ExternalSeverity: External Severity
+  ExternalSeverity:
+    - External Severity
+    - Observeit Severity
   SourceMAC:
     - SourceMAC
     - MAC
@@ -73,6 +75,6 @@ field_mapping:
   SourceUserName: SourceUserName
   url_category: XForceCategoryByURL
   EventSeverity: EventSeverity
-  Source: 
+  Source:
     - Source
-    - source
+    - source

uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml

@@ -11,10 +11,13 @@ default_log_source:
 field_mapping:
   src-ip:
     - sourceip
+    - sourceIP
+    - SourceIP
     - SrcHost
     - LocalHost
     - Source
     - NetworkView
+    - HostName
   src-port:
     - sourceport
     - SrcPort

uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml

@@ -11,9 +11,12 @@ default_log_source:
   category: 8110
 
 field_mapping:
-  CommandLine: Command
+  CommandLine:
+    - Command
+    - ASACommand
   Image: Process Path
   ParentCommandLine: Parent Command
   ParentImage: Parent Process Path
   User: username
-  LogonId: Logon ID
+  LogonId: Logon ID
+  EventID: ASASyslogCode