Merge pull request #138 from UncoderIO/gis-8002

For raw log fields add field type. Add and improve mappings

Author: saltar-ua

uncoder-core/app/translator/core/mapping.py

@@ -72,7 +72,7 @@ def __init__(
         source_id: str,
         log_source_signature: _LogSourceSignatureType = None,
         fields_mapping: Optional[FieldsMapping] = None,
-        raw_log_fields: Optional[list] = None,
+        raw_log_fields: Optional[dict] = None,
     ):
         self.source_id = source_id
         self.log_source_signature = log_source_signature
@@ -103,7 +103,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
                     continue
 
             field_mappings_dict = mapping_dict.get("field_mapping", {})
-            raw_log_fields = mapping_dict.get("raw_log_fields", [])
+            raw_log_fields = mapping_dict.get("raw_log_fields", {})
             field_mappings_dict.update({field: field for field in raw_log_fields})
             fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict)
             self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping)

uncoder-core/app/translator/core/render.py

@@ -21,6 +21,7 @@
 from typing import Optional, Union
 
 from app.translator.const import DEFAULT_VALUE_TYPE
+from app.translator.core.context_vars import return_only_first_query_ctx_var
 from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
 from app.translator.core.custom_types.values import ValueType
 from app.translator.core.escape_manager import EscapeManager
@@ -192,7 +193,7 @@ class PlatformQueryRender(QueryRender):
     field_value_map = BaseQueryFieldValue(or_token=or_token)
 
     query_pattern = "{table} {query} {functions}"
-    raw_log_field_pattern: str = None
+    raw_log_field_pattern_map: dict = None
 
     def __init__(self):
         self.operator_map = {
@@ -283,6 +284,7 @@ def finalize_query(
         **kwargs,  # noqa: ARG002
     ) -> str:
         query = self.query_pattern.format(prefix=prefix, query=query, functions=functions).strip()
+
         query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query)
         if not_supported_functions:
             rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
@@ -323,6 +325,16 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer)
             prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info
         )
 
+    def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
+        if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type):
+            return raw_log_field_pattern.pattern.format(field=field)
+
+    def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[str]:
+        if self.raw_log_field_pattern_map is None:
+            return
+        if raw_log_field_type := source_mapping.raw_log_fields.get(field):
+            return self.process_raw_log_field(field=field, field_type=raw_log_field_type)
+
     def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str:
         defined_raw_log_fields = []
         for field in fields:
@@ -334,10 +346,8 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
                 )
             if not mapped_field and self.is_strict_mapping:
                 raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
-            if mapped_field not in source_mapping.raw_log_fields:
-                continue
-            field_prefix = self.raw_log_field_pattern.format(field=mapped_field)
-            defined_raw_log_fields.append(field_prefix)
+            if field_prefix := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
+                defined_raw_log_fields.append(field_prefix)
         return "\n".join(set(defined_raw_log_fields))
 
     def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
@@ -368,6 +378,8 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
                     meta_info=query_container.meta_info,
                     source_mapping=source_mapping,
                 )
+                if return_only_first_query_ctx_var.get() is True:
+                    return finalized_query
                 queries_map[source_mapping.source_id] = finalized_query
         if not queries_map and errors:
             raise errors[0]

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml

@@ -0,0 +1,35 @@
+platform: Palo Alto XSIAM
+source: aws_cloudtrail
+
+
+default_log_source:
+  dataset: amazon_aws_raw
+
+field_mapping:
+  eventSource: eventSource
+  eventName: eventName
+  errorCode: errorCode
+  errorMessage: errorMessage
+  eventType: eventType
+  requestParameters: requestParameters
+  responseElements: responseElements
+  status: status
+  terminatingRuleId: terminatingRuleId
+  userAgent: userAgent
+  AdditionalEventData.MFAUsed: additionalEventData.MFAUsed
+
+
+raw_log_fields:
+  additionalEventData.MFAUsed: object
+  requestParameters.ipPermissions.items.ipRanges.items.cidrIP: object
+  requestParameters.ipPermissions.items.ipRanges.items.fromPort: object
+  requestParameters.attribute: object
+  requestParameters.userData: list
+  responseElements.ConsoleLogin: object
+  responseElements.pendingModifiedValues.masterUserPassword: object
+  responseElements.publiclyAccessible: object
+  userIdentity.arn: object
+  userIdentity.principalId: object
+  userIdentity.sessionContext.sessionIssuer.type: object
+  userIdentity.type: object
+  userIdentity.userName: object

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_eks.yml

@@ -0,0 +1,25 @@
+platform: Palo Alto XSIAM
+source: aws_eks
+
+
+default_log_source:
+  dataset: amazon_aws_raw
+
+field_mapping:
+  aws_node_type: aws_node_type
+  requestURI: requestURI
+  stage: stage
+  verb: verb
+
+
+raw_log_fields:
+  annotations.authorization.k8s.io\/decision: object
+  annotations.podsecuritypolicy.policy.k8s.io\/admit-policy: object
+  objectRef.namespace: object
+  objectRef.resource: object
+  objectRef.subresource: object
+  requestObject.rules.resources: object
+  requestObject.rules.verbs: object
+  requestObject.spec.containers.image: object
+  user.groups: object
+  user.username: object

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_aadnoninteractiveusersigninlogs.yml

@@ -0,0 +1,16 @@
+platform: Palo Alto XSIAM
+source: azure_aadnoninteractiveusersigninlogs
+
+
+default_log_source:
+  dataset: msft_azure_raw
+
+field_mapping:
+  UserAgent: properties.userAgent
+  Type: properties.type
+  AuthenticationProcessingDetails: properties.authenticationProcessingDetails
+
+raw_log_fields:
+ properties.userAgent: object
+ properties.type: object
+ properties.authenticationProcessingDetails: object

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml

@@ -9,16 +9,16 @@ field_mapping:
   Provider_Name: provider_name
 
 raw_log_fields:
-  - src_ip
-  - source
-  - additional_information
-  - EventData
-  - Channel
-  - statement
-  - Faulting application path
-  - object_name
-  - class_type
-  - action_id
-  - Data
-  - Message
-  - Level
+  src_ip: regex
+  source: regex
+  additional_information: regex
+  EventData: regex
+  Channel: regex
+  statement: regex
+  Faulting application path: regex
+  object_name: regex
+  class_type: regex
+  action_id: regex
+  Data: regex
+  Message: regex
+  Level: regex

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml

@@ -55,4 +55,4 @@ field_mapping:
 
 
 raw_log_fields:
-  - Initiated
+  Initiated: regex

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml

@@ -8,5 +8,5 @@ field_mapping:
   EventID: action_evtlog_event_id
 
 raw_log_fields:
-  - PipeName
-  - Image
+  PipeName: regex
+  Image: regex

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml

@@ -10,10 +10,10 @@ field_mapping:
 
 
 raw_log_fields:
-  - CommandLine
-  - ScriptBlockText
-  - Payload
-  - HostApplication
-  - ContextInfo
-  - HostName
-  - EngineVersion
+  CommandLine: regex
+  ScriptBlockText: regex
+  Payload: regex
+  HostApplication: regex
+  ContextInfo: regex
+  HostName: regex
+  EngineVersion: regex

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml

@@ -6,14 +6,15 @@ default_log_source:
 
 field_mapping:
   User: action_process_username
+  SourceUser: action_process_username
 
 raw_log_fields:
-  - SourceProcessGUID
-  - SourceProcessId
-  - SourceThreadId
-  - SourceImage
-  - TargetProcessGUID
-  - TargerProcessId
-  - TargetImage
-  - GrantedAccess
-  - CallTrace
+  SourceProcessGUID: regex
+  SourceProcessId: regex
+  SourceThreadId: regex
+  SourceImage: regex
+  TargetProcessGUID: regex
+  TargerProcessId: regex
+  TargetImage: regex
+  GrantedAccess: regex
+  CallTrace: regex