Merge branch 'prod' into 'gis-case-insensitive-sigma-mapping'

# Conflicts:
#   app/translator/platforms/palo_alto/renders/cortex_xsiam.py

Author: tarnopolskyi

uncoder-core/app/translator/core/context_vars.py

@@ -1,4 +1,4 @@
 from contextvars import ContextVar
 
 return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False)
-"""Set to True to return ony first query if rendered multiple options"""
+"""Set to True to return only first query if rendered multiple options"""

uncoder-core/app/translator/core/custom_types/time.py

@@ -0,0 +1,23 @@
+from app.translator.tools.custom_enum import CustomEnum
+
+
+class TimeFrameType(CustomEnum):
+    years = "years"
+    months = "months"
+    days = "days"
+    hours = "hours"
+    minutes = "minutes"
+
+
+class TimePartType(CustomEnum):
+    day = "day"
+    day_of_week = "day_of_week"
+    day_of_year = "day_of_year"
+    hour = "hour"
+    microsecond = "microsecond"
+    millisecond = "millisecond"
+    minute = "minute"
+    month = "month"
+    quarter = "quarter"
+    second = "second"
+    year = "year"

uncoder-core/app/translator/core/exceptions/core.py

@@ -1,3 +1,6 @@
+from typing import Optional
+
+
 class NotImplementedException(BaseException):
     ...
 
@@ -7,8 +10,19 @@ class BasePlatformException(BaseException):
 
 
 class StrictPlatformException(BasePlatformException):
-    def __init__(self, platform_name: str, field_name: str):
-        message = f"Platform {platform_name} has strict mapping. Source field {field_name} has no mapping."
+    field_name: str = None
+
+    def __init__(
+        self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
+    ):
+        message = (
+            f"Platform {platform_name} has strict mapping. "
+            f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
+            f" Mapping file: {mapping}."
+            if mapping
+            else ""
+        )
+        self.field_name = field_name
         super().__init__(message)
 
 

uncoder-core/app/translator/core/mapping.py

@@ -20,6 +20,10 @@ def is_suitable(self, *args, **kwargs) -> bool:
     def __str__(self) -> str:
         raise NotImplementedError("Abstract method")
 
+    @property
+    def default_source(self) -> dict:
+        return self._default_source
+
 
 class FieldMapping:
     def __init__(self, generic_field_name: str, platform_field_name: str):

uncoder-core/app/translator/core/parser.py

@@ -15,6 +15,7 @@
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 -----------------------------------------------------------------
 """
+
 import re
 from abc import ABC, abstractmethod
 from typing import Union

uncoder-core/app/translator/core/render.py

@@ -16,6 +16,7 @@
 limitations under the License.
 -----------------------------------------------------------------
 """
+
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from typing import ClassVar, Optional, Union
@@ -196,10 +197,10 @@ class PlatformQueryRender(QueryRender):
     not_token = "not"
 
     group_token = "(%s)"
+    query_parts_delimiter = " "
 
     field_value_map = BaseQueryFieldValue(or_token=or_token)
 
-    query_pattern = "{table} {query} {functions}"
     raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
 
     def __init__(self):
@@ -210,9 +211,9 @@ def __init__(self):
             LogicalOperatorType.NOT: f" {self.not_token} ",
         }
 
-    def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str:  # noqa: ARG002
-        if str(log_source_signature):
-            return f"{log_source_signature!s} {self.and_token}"
+    def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str:  # noqa: ARG002
+        if log_source_signature and str(log_source_signature):
+            return f"{log_source_signature} {self.and_token}"
         return ""
 
     def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions:
@@ -262,8 +263,14 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
 
     def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str:
         result_values = []
+        unmapped_fields = set()
         for token in tokens:
-            result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
+            try:
+                result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
+            except StrictPlatformException as err:
+                unmapped_fields.add(err.field_name)
+        if unmapped_fields:
+            raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
         return "".join(result_values)
 
     def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str:
@@ -280,6 +287,14 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) ->
             query = f"{query}\n\n{query_meta_info}"
         return query
 
+    @staticmethod
+    def _finalize_search_query(query: str) -> str:
+        return query
+
+    def _join_query_parts(self, prefix: str, query: str, functions: str) -> str:
+        parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions]))
+        return self.query_parts_delimiter.join(parts)
+
     def finalize_query(
         self,
         prefix: str,
@@ -291,8 +306,7 @@ def finalize_query(
         *args,  # noqa: ARG002
         **kwargs,  # noqa: ARG002
     ) -> str:
-        query = self.query_pattern.format(prefix=prefix, query=query, functions=functions).strip()
-
+        query = self._join_query_parts(prefix, query, functions)
         query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query)
         if not_supported_functions:
             rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
@@ -335,15 +349,15 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer)
 
     def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
         if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type):
-            return raw_log_field_pattern.pattern.format(field=field)
+            return raw_log_field_pattern.format(field=field)
 
     def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]:
         if isinstance(field, list):
-            list_of_prefix = []
+            prefix_list = []
             for f in field:
-                if prepared_prefix := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping):
-                    list_of_prefix.extend(prepared_prefix)
-            return list_of_prefix
+                if _prefix_list := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping):
+                    prefix_list.extend(_prefix_list)
+            return prefix_list
         if raw_log_field_type := source_mapping.raw_log_fields.get(field):
             return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
 
@@ -360,9 +374,11 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
                 )
             if not mapped_field and self.is_strict_mapping:
                 raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
-            if field_prefix := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
-                defined_raw_log_fields.extend(field_prefix)
-        return "\n".join(set(defined_raw_log_fields))
+            if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
+                for prefix in prefix_list:
+                    if prefix not in defined_raw_log_fields:
+                        defined_raw_log_fields.append(prefix)
+        return "\n".join(defined_raw_log_fields)
 
     def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
         queries_map = {}
@@ -377,7 +393,7 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
                     defined_raw_log_fields = self.generate_raw_log_fields(
                         fields=query_container.meta_info.query_fields, source_mapping=source_mapping
                     )
-                    prefix += f"\n{defined_raw_log_fields}\n"
+                    prefix += f"\n{defined_raw_log_fields}"
                 result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
             except StrictPlatformException as err:
                 errors.append(err)

uncoder-core/app/translator/core/render_cti.py

@@ -17,7 +17,6 @@
 -----------------------------------------------------------------
 """
 
-
 from app.translator.core.models.iocs import IocsChunkValue
 from app.translator.core.models.platform_details import PlatformDetails
 

uncoder-core/app/translator/core/str_value_manager.py

@@ -16,6 +16,7 @@
 limitations under the License.
 -----------------------------------------------------------------
 """
+
 from typing import ClassVar, Optional, TypeVar, Union
 
 from app.translator.core.custom_types.values import ValueType

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml

@@ -32,4 +32,5 @@ raw_log_fields:
   userIdentity.principalId: object
   userIdentity.sessionContext.sessionIssuer.type: object
   userIdentity.type: object
-  userIdentity.userName: object
+  userIdentity.userName: object
+  requestParameters.publiclyAccessible: object

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_signinlogs.yml

@@ -0,0 +1,46 @@
+platform: Palo Alto XSIAM
+source: azure_signinlogs
+
+
+default_log_source:
+  dataset: msft_azure_raw
+
+field_mapping:
+  AppDisplayName: properties.appDisplayName
+  AppId: properties.appId
+  AuthenticationRequirement: properties.authenticationRequirement
+  Category: properties.category
+  ConditionalAccessStatus: properties.conditionalAccessStatus
+  DeviceDetail: properties.deviceDetail
+  IsInteractive: properties.isInteractive
+  NetworkLocationDetails: properties.networkLocationDetails
+  ResourceDisplayName: properties.resourceDisplayName
+  ResourceIdentity: properties.resourceIdentity
+  ResultDescription: properties.resultDescription
+  ResultType: properties.resultType
+  Status.errorCode: properties.status.errorCode
+  Status: properties.status
+  Status.failureReason: properties.status.failureReason
+  TokenIssuerType: properties.tokenIssuerType
+  UserAgent: properties.userAgent
+  UserPrincipalName: properties.userPrincipalName
+
+raw_log_fields:
+  properties.appDisplayName: object
+  properties.appId: object
+  properties.authenticationRequirement: object
+  properties.category: object
+  properties.conditionalAccessStatus: object
+  properties.deviceDetail: object
+  properties.isInteractive: object
+  properties.networkLocationDetails: object
+  properties.resourceDisplayName: object
+  properties.resourceIdentity: object
+  properties.resultDescription: object
+  properties.resultType: object
+  properties.status.errorCode: object
+  properties.status: object
+  properties.status.failureReason: object
+  properties.tokenIssuerType: object
+  properties.userAgent: object
+  properties.userPrincipalName: object