Skip to content

Commit 83c12c8

Browse files
alexvolhaalexvolha
committed
render unmapped fields comment
1 parent 14ec9a0 commit 83c12c8

File tree

15 files changed

+30
-83
lines changed

15 files changed

+30
-83
lines changed

‎uncoder-core/app/translator/core/exceptions/core.py‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,14 @@ class BasePlatformException(BaseException):
1010

1111

1212
class StrictPlatformException(BasePlatformException):
13-
field_name: str = None
14-
15-
def __init__(
16-
self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
17-
):
13+
def __init__(self, platform_name: str, fields: list[str], mapping: Optional[str] = None):
1814
message = (
1915
f"Platform {platform_name} has strict mapping. "
20-
f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
16+
f"Source fields: {', '.join(fields)} have no mapping."
2117
f" Mapping file: {mapping}."
2218
if mapping
2319
else ""
2420
)
25-
self.field_name = field_name
2621
super().__init__(message)
2722

2823

‎uncoder-core/app/translator/core/mapping.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -158,17 +158,18 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
158158
def default_mapping(self) -> SourceMapping:
159159
return self._source_mappings[DEFAULT_MAPPING_NAME]
160160

161-
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[Field]:
162-
not_mapped = []
161+
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
162+
unmapped = []
163163
for field in field_tokens:
164164
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
165165
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
166-
if not mapped_field:
167-
if self.is_strict_mapping:
168-
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
169-
not_mapped.append(field)
166+
if not mapped_field and field.source_name not in unmapped:
167+
unmapped.append(field.source_name)
170168

171-
return not_mapped
169+
if self.is_strict_mapping and unmapped:
170+
raise StrictPlatformException(platform_name=self.details.name, fields=unmapped)
171+
172+
return unmapped
172173

173174
@staticmethod
174175
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -207,10 +207,9 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions:
207207

208208
return query
209209

210-
def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[Field]]) -> str:
210+
def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[str]]) -> str:
211211
if fields:
212-
joined = ", ".join(field.source_name for field in fields)
213-
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{joined}")
212+
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}")
214213
return query
215214

216215
def wrap_with_comment(self, value: str) -> str:
@@ -256,7 +255,7 @@ def generate_functions(self, functions: list[Function], source_mapping: SourceMa
256255
def map_predefined_field(self, predefined_field: PredefinedField) -> str:
257256
if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
258257
if self.mappings.is_strict_mapping:
259-
raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name)
258+
raise StrictPlatformException(platform_name=self.details.name, fields=[predefined_field.name])
260259

261260
return predefined_field.name
262261

@@ -309,14 +308,9 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
309308

310309
def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str:
311310
result_values = []
312-
unmapped_fields = set()
313311
for token in tokens:
314-
try:
315-
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
316-
except StrictPlatformException as err:
317-
unmapped_fields.add(err.field_name)
318-
if unmapped_fields:
319-
raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
312+
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
313+
320314
return "".join(result_values)
321315

322316
def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str:
@@ -349,7 +343,7 @@ def finalize_query(
349343
meta_info: Optional[MetaInfoContainer] = None,
350344
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
351345
not_supported_functions: Optional[list] = None,
352-
unmapped_fields: Optional[list[Field]] = None,
346+
unmapped_fields: Optional[list[str]] = None,
353347
*args, # noqa: ARG002
354348
**kwargs, # noqa: ARG002
355349
) -> str:
@@ -418,7 +412,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
418412
generic_field_name=generic_field_name
419413
)
420414
if not mapped_field and self.mappings.is_strict_mapping:
421-
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
415+
raise StrictPlatformException(platform_name=self.details.name, fields=[field.source_name])
422416
if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
423417
for prefix in prefix_list:
424418
if prefix not in defined_raw_log_fields:

‎uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
from app.translator.core.mapping import SourceMapping
2525
from app.translator.core.models.platform_details import PlatformDetails
2626
from app.translator.core.models.query_container import MetaInfoContainer
27-
from app.translator.core.models.query_tokens.field import Field
2827
from app.translator.managers import render_manager
2928
from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details
3029
from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_rule_mappings
@@ -112,7 +111,7 @@ def finalize_query(
112111
meta_info: Optional[MetaInfoContainer] = None,
113112
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
114113
not_supported_functions: Optional[list] = None, # ,
115-
unmapped_fields: Optional[list[Field]] = None,
114+
unmapped_fields: Optional[list[str]] = None,
116115
*args, # noqa: ARG002
117116
**kwargs, # noqa: ARG002
118117
) -> str:

‎uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
from app.translator.core.mitre import MitreConfig
2626
from app.translator.core.models.platform_details import PlatformDetails
2727
from app.translator.core.models.query_container import MetaInfoContainer
28-
from app.translator.core.models.query_tokens.field import Field
2928
from app.translator.managers import render_manager
3029
from app.translator.platforms.base.lucene.mapping import LuceneMappings
3130
from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_DETECTION_RULE, elasticsearch_rule_details
@@ -88,7 +87,7 @@ def finalize_query(
8887
meta_info: Optional[MetaInfoContainer] = None,
8988
source_mapping: Optional[SourceMapping] = None,
9089
not_supported_functions: Optional[list] = None,
91-
unmapped_fields: Optional[list[Field]] = None,
90+
unmapped_fields: Optional[list[str]] = None,
9291
*args, # noqa: ARG002
9392
**kwargs, # noqa: ARG002
9493
) -> str:

‎uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
from app.translator.core.mapping import SourceMapping
2424
from app.translator.core.models.platform_details import PlatformDetails
2525
from app.translator.core.models.query_container import MetaInfoContainer
26-
from app.translator.core.models.query_tokens.field import Field
2726
from app.translator.managers import render_manager
2827
from app.translator.platforms.base.lucene.mapping import LuceneMappings
2928
from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_ALERT, elastalert_details
@@ -61,7 +60,7 @@ def finalize_query(
6160
meta_info: Optional[MetaInfoContainer] = None,
6261
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
6362
not_supported_functions: Optional[list] = None,
64-
unmapped_fields: Optional[list[Field]] = None,
63+
unmapped_fields: Optional[list[str]] = None,
6564
*args, # noqa: ARG002
6665
**kwargs, # noqa: ARG002
6766
) -> str:

‎uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
from app.translator.core.mapping import SourceMapping
2525
from app.translator.core.models.platform_details import PlatformDetails
2626
from app.translator.core.models.query_container import MetaInfoContainer
27-
from app.translator.core.models.query_tokens.field import Field
2827
from app.translator.managers import render_manager
2928
from app.translator.platforms.base.lucene.mapping import LuceneMappings
3029
from app.translator.platforms.elasticsearch.const import KIBANA_RULE, KIBANA_SEARCH_SOURCE_JSON, kibana_rule_details
@@ -57,7 +56,7 @@ def finalize_query(
5756
meta_info: Optional[MetaInfoContainer] = None,
5857
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
5958
not_supported_functions: Optional[list] = None,
60-
unmapped_fields: Optional[list[Field]] = None,
59+
unmapped_fields: Optional[list[str]] = None,
6160
*args, # noqa: ARG002
6261
**kwargs, # noqa: ARG002
6362
) -> str:

‎uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
from app.translator.core.mapping import SourceMapping
2525
from app.translator.core.models.platform_details import PlatformDetails
2626
from app.translator.core.models.query_container import MetaInfoContainer
27-
from app.translator.core.models.query_tokens.field import Field
2827
from app.translator.managers import render_manager
2928
from app.translator.platforms.base.lucene.mapping import LuceneMappings
3029
from app.translator.platforms.elasticsearch.const import XPACK_WATCHER_RULE, xpack_watcher_details
@@ -57,7 +56,7 @@ def finalize_query(
5756
meta_info: Optional[MetaInfoContainer] = None,
5857
source_mapping: Optional[SourceMapping] = None,
5958
not_supported_functions: Optional[list] = None,
60-
unmapped_fields: Optional[list[Field]] = None,
59+
unmapped_fields: Optional[list[str]] = None,
6160
*args, # noqa: ARG002
6261
**kwargs, # noqa: ARG002
6362
) -> str:

‎uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
from app.translator.core.mapping import SourceMapping
2727
from app.translator.core.models.platform_details import PlatformDetails
2828
from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer
29-
from app.translator.core.models.query_tokens.field import Field
3029
from app.translator.core.models.query_tokens.field_value import FieldValue
3130
from app.translator.core.models.query_tokens.identifier import Identifier
3231
from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
@@ -304,7 +303,7 @@ def finalize_query(
304303
meta_info: Optional[MetaInfoContainer] = None,
305304
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
306305
not_supported_functions: Optional[list] = None,
307-
unmapped_fields: Optional[list[Field]] = None,
306+
unmapped_fields: Optional[list[str]] = None,
308307
fields: Optional[set[str]] = None,
309308
*args, # noqa: ARG002
310309
**kwargs, # noqa: ARG002

‎uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
from app.translator.core.mapping import SourceMapping
2626
from app.translator.core.models.platform_details import PlatformDetails
2727
from app.translator.core.models.query_container import MetaInfoContainer
28-
from app.translator.core.models.query_tokens.field import Field
2928
from app.translator.managers import render_manager
3029
from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details
3130
from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager
@@ -66,7 +65,7 @@ def finalize_query(
6665
meta_info: Optional[MetaInfoContainer] = None,
6766
source_mapping: Optional[SourceMapping] = None,
6867
not_supported_functions: Optional[list] = None,
69-
unmapped_fields: Optional[list[Field]] = None,
68+
unmapped_fields: Optional[list[str]] = None,
7069
*args, # noqa: ARG002
7170
**kwargs, # noqa: ARG002
7271
) -> str:

0 commit comments

Comments
 (0)