Skip to content

Commit 6f1173b

Browse files
committed
merge main
2 parents 9a9f093 + 8b77550 commit 6f1173b

File tree

26 files changed

+221
-151
lines changed

26 files changed

+221
-151
lines changed

uncoder-core/app/routers/translate.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from fastapi import APIRouter, Body
22

33
from app.models.translation import InfoMessage, OneTranslationData, Platform, TranslatorPlatforms
4+
from app.translator.core.context_vars import return_only_first_query_ctx_var
45
from app.translator.cti_translator import CTITranslator
56
from app.translator.translator import Translator
67

@@ -15,7 +16,9 @@ def translate_one(
1516
source_platform_id: str = Body(..., embed=True),
1617
target_platform_id: str = Body(..., embed=True),
1718
text: str = Body(..., embed=True),
19+
return_only_first_query: bool = False,
1820
) -> OneTranslationData:
21+
return_only_first_query_ctx_var.set(return_only_first_query)
1922
status, data = translator.translate_one(text=text, source=source_platform_id, target=target_platform_id)
2023
if status:
2124
return OneTranslationData(status=status, translation=data, target_platform_id=target_platform_id)
@@ -27,8 +30,11 @@ def translate_one(
2730
@st_router.post("/translate/all", tags=["translator"], description="Generate all translations")
2831
@st_router.post("/translate/all/", include_in_schema=False)
2932
def translate_all(
30-
source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True)
33+
source_platform_id: str = Body(..., embed=True),
34+
text: str = Body(..., embed=True),
35+
return_only_first_query: bool = False,
3136
) -> list[OneTranslationData]:
37+
return_only_first_query_ctx_var.set(return_only_first_query)
3238
result = translator.translate_all(text=text, source=source_platform_id)
3339
translations = []
3440
for platform_result in result:

uncoder-core/app/translator/core/custom_types/functions.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ class FunctionType(CustomEnum):
99
min = "min"
1010
sum = "sum"
1111

12-
divide = "divide"
12+
values = "values"
1313

1414
earliest = "earliest"
1515
latest = "latest"
1616

17+
divide = "divide"
18+
1719
lower = "lower"
20+
split = "split"
1821
upper = "upper"
1922

20-
compare_boolean = "compare_boolean"
21-
23+
array_length = "array_length"
24+
compare = "compare"
25+
extract_time = "extract_time"
2226
ipv4_is_in_range = "ipv4_is_in_range"
2327

2428
bin = "bin"
@@ -30,4 +34,6 @@ class FunctionType(CustomEnum):
3034
stats = "stats"
3135
table = "table"
3236
timeframe = "timeframe"
33-
values = "values"
37+
union = "union"
38+
39+
aggregation_data_function = "aggregation_data_function"

uncoder-core/app/translator/core/functions.py

Lines changed: 75 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
import re
2222
from abc import ABC, abstractmethod
2323
from dataclasses import dataclass
24-
from functools import cached_property
25-
from typing import TYPE_CHECKING, Any, Optional
24+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
2625

27-
from app.translator.core.exceptions.functions import InvalidFunctionSignature, NotSupportedFunctionException
26+
from app.translator.core.exceptions.functions import NotSupportedFunctionException
2827
from app.translator.core.mapping import SourceMapping
29-
from app.translator.core.models.field import Field
28+
from app.translator.core.models.field import Alias, Field
3029
from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
30+
from app.translator.tools.utils import execute_module
3131
from settings import INIT_FUNCTIONS
3232

3333
if TYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
4141

4242

4343
class BaseFunctionParser(ABC):
44+
function_names_map: ClassVar[dict[str, str]] = {}
45+
functions_group_name: str = None
46+
manager: PlatformFunctionsManager = None
47+
48+
def set_functions_manager(self, manager: PlatformFunctionsManager) -> BaseFunctionParser:
49+
self.manager = manager
50+
return self
51+
4452
@abstractmethod
4553
def parse(self, *args, **kwargs) -> Function:
4654
raise NotImplementedError
@@ -73,21 +81,25 @@ def parse(self, func_body: str, raw: str) -> Function:
7381

7482

7583
class FunctionRender(ABC):
84+
function_names_map: ClassVar[dict[str, str]] = {}
85+
order_to_render: int = 0
86+
in_query_render: bool = False
87+
render_to_prefix: bool = False
88+
manager: PlatformFunctionsManager = None
89+
90+
def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRender:
91+
self.manager = manager
92+
return self
93+
7694
@abstractmethod
7795
def render(self, function: Function, source_mapping: SourceMapping) -> str:
7896
raise NotImplementedError
7997

8098
@staticmethod
81-
def concat_kwargs(kwargs: dict[str, str]) -> str:
82-
result = ""
83-
for key, value in kwargs.items():
84-
if value:
85-
result = f"{result}, {key}={value}" if result else f"{key}={value}"
99+
def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
100+
if isinstance(field, Alias):
101+
return field.name
86102

87-
return result
88-
89-
@staticmethod
90-
def map_field(field: Field, source_mapping: SourceMapping) -> str:
91103
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
92104
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
93105
if isinstance(mapped_field, list):
@@ -97,23 +109,48 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
97109

98110

99111
class PlatformFunctionsManager:
112+
platform_functions: PlatformFunctions = None
113+
100114
def __init__(self):
101-
self._parsers_map: dict[str, HigherOrderFunctionParser] = {}
102-
self._renders_map: dict[str, FunctionRender] = {}
103-
self._in_query_renders_map: dict[str, FunctionRender] = {}
104-
self._names_map: dict[str, str] = {}
105-
self._order_to_render: dict[str, int] = {}
106-
self._render_to_prefix_functions: list[str] = []
107-
108-
def post_init_configure(self, platform_render: PlatformQueryRender) -> None:
109-
raise NotImplementedError
115+
# {platform_func_name: HigherOrderFunctionParser}
116+
self._hof_parsers_map: dict[str, HigherOrderFunctionParser] = {}
117+
self._parsers_map: dict[str, FunctionParser] = {} # {platform_func_name: FunctionParser}
118+
119+
self._renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
120+
self._in_query_renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
121+
self._order_to_render: dict[str, int] = {} # {generic_func_name: int}
122+
123+
def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]:
124+
render = render_class()
125+
render.manager = self
126+
for generic_function_name in render.function_names_map:
127+
self._renders_map[generic_function_name] = render
128+
self._order_to_render[generic_function_name] = render.order_to_render
129+
if render.in_query_render:
130+
self._in_query_renders_map[generic_function_name] = render
131+
132+
return render_class
133+
134+
def register_parser(self, parser_class: type[BaseFunctionParser]) -> type[BaseFunctionParser]:
135+
parser = parser_class()
136+
parser.manager = self
137+
parsers_map = self._hof_parsers_map if isinstance(parser, HigherOrderFunctionParser) else self._parsers_map
138+
for platform_function_name in parser.function_names_map:
139+
parsers_map[platform_function_name] = parser
140+
141+
if parser.functions_group_name:
142+
parsers_map[parser.functions_group_name] = parser
143+
144+
return parser_class
145+
146+
def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
147+
if INIT_FUNCTIONS and (parser := self._hof_parsers_map.get(platform_func_name)):
148+
return parser
110149

111-
@cached_property
112-
def _inverted_names_map(self) -> dict[str, str]:
113-
return {value: key for key, value in self._names_map.items()}
150+
raise NotSupportedFunctionException
114151

115-
def get_parser(self, generic_func_name: str) -> HigherOrderFunctionParser:
116-
if INIT_FUNCTIONS and (parser := self._parsers_map.get(generic_func_name)):
152+
def get_parser(self, platform_func_name: str) -> FunctionParser:
153+
if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)):
117154
return parser
118155

119156
raise NotSupportedFunctionException
@@ -130,56 +167,29 @@ def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
130167

131168
raise NotSupportedFunctionException
132169

133-
def get_generic_func_name(self, platform_func_name: str) -> Optional[str]:
134-
if INIT_FUNCTIONS and (generic_func_name := self._names_map.get(platform_func_name)):
135-
return generic_func_name
136-
137-
raise NotSupportedFunctionException
138-
139-
def get_platform_func_name(self, generic_func_name: str) -> Optional[str]:
140-
if INIT_FUNCTIONS:
141-
return self._inverted_names_map.get(generic_func_name)
142-
143170
@property
144171
def order_to_render(self) -> dict[str, int]:
145172
if INIT_FUNCTIONS:
146173
return self._order_to_render
147174

148175
return {}
149176

150-
@property
151-
def render_to_prefix_functions(self) -> list[str]:
152-
if INIT_FUNCTIONS:
153-
return self._render_to_prefix_functions
154-
155-
return []
156-
157177

158178
class PlatformFunctions:
179+
dir_path: str = None
180+
platform_query_render: PlatformQueryRender = None
159181
manager: PlatformFunctionsManager = PlatformFunctionsManager()
182+
160183
function_delimiter = "|"
161184

162-
def parse(self, query: str) -> ParsedFunctions:
163-
parsed = []
164-
not_supported = []
165-
invalid = []
166-
functions = query.split(self.function_delimiter)
167-
for func in functions:
168-
split_func = func.strip().split(" ")
169-
func_name, func_body = split_func[0], " ".join(split_func[1:])
170-
try:
171-
func_parser = self.manager.get_parser(self.manager.get_generic_func_name(func_name))
172-
parsed.append(func_parser.parse(func_body, func))
173-
except NotSupportedFunctionException:
174-
not_supported.append(func)
175-
except InvalidFunctionSignature:
176-
invalid.append(func)
185+
def __init__(self):
186+
self.manager.platform_functions = self
187+
if self.dir_path:
188+
execute_module(f"{self.dir_path}/parsers/__init__.py")
189+
execute_module(f"{self.dir_path}/renders/__init__.py")
177190

178-
return ParsedFunctions(
179-
functions=parsed,
180-
not_supported=[self.wrap_function_with_delimiter(func) for func in not_supported],
181-
invalid=invalid,
182-
)
191+
def parse(self, query: str) -> ParsedFunctions: # noqa: ARG002
192+
return ParsedFunctions()
183193

184194
def _sort_functions_to_render(self, functions: list[Function]) -> list[Function]:
185195
return sorted(functions, key=lambda func: self.manager.order_to_render.get(func.name, 0))
@@ -193,7 +203,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
193203
try:
194204
func_render = self.manager.get_render(func.name)
195205
_rendered = func_render.render(func, source_mapping)
196-
if func.name in self.manager.render_to_prefix_functions:
206+
if func_render.render_to_prefix:
197207
rendered_prefix += _rendered
198208
else:
199209
rendered += self.wrap_function_with_delimiter(_rendered)

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ field_mapping:
1010
#dns-record: dns-record
1111
dns_query_name: xdm.network.dns.dns_question.name
1212
QueryName: xdm.network.dns.dns_question.name
13-
query: xdm.network.dns.dns_question.name
13+
query: xdm.network.dns.dns_question.name
14+
dns-record-type: xdm.network.dns.dns_question.type

uncoder-core/app/translator/mappings/platforms/qradar/default.yml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ field_mapping:
3535
- userName
3636
- EventUserName
3737
CommandLine: Command
38-
Protocol: IPProtocol
38+
Protocol:
39+
- IPProtocol
40+
- protocol
3941
Application:
4042
- Application
4143
- application
@@ -61,6 +63,7 @@ field_mapping:
6163
SourceMAC:
6264
- SourceMAC
6365
- MAC
66+
- sourceMAC
6467
DestinationMAC: DestinationMAC
6568
SourceOS:
6669
- SourceOS
@@ -69,4 +72,7 @@ field_mapping:
6972
TargetUserName: DestinationUserName
7073
SourceUserName: SourceUserName
7174
url_category: XForceCategoryByURL
72-
EventSeverity: EventSeverity
75+
EventSeverity: EventSeverity
76+
Source:
77+
- Source
78+
- source

uncoder-core/app/translator/mappings/platforms/qradar/dns.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ field_mapping:
1212
dns-query: URL
1313
parent-domain: parent-domain
1414
dns-answer: dns-answer
15-
dns-record: URL
15+
dns-record: URL
16+
dns-record-type: DNSRecordType

uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ field_mapping:
2424
cs-host:
2525
- UrlHost
2626
- URL Host
27+
- URL Domain
2728
cs-referrer:
2829
- URL Referrer
2930
- Referrer URL

uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ field_mapping:
4141
LinkName: LinkName
4242
MemberName: MemberName
4343
MemberSid: MemberSid
44-
NewProcessName: Process Name
44+
NewProcessName:
45+
- Process Name
46+
- New Process Name
4547
ObjectClass: ObjectClass
4648
ObjectName:
4749
- Object Name
@@ -122,6 +124,7 @@ field_mapping:
122124
ServiceFileName:
123125
- Service Filename
124126
- ServiceFileName
127+
- Service File Name
125128
SecurityDescriptor: SecurityDescriptor
126129
ServiceName: Service Name
127130
ShareName:
Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
1-
import importlib.util
21
import os
32

3+
from app.translator.tools.utils import execute_module
44
from const import PLATFORMS_PATH
55

66

7-
def init_platforms():
7+
def init_platforms() -> None:
88
for platform in [f for f in os.listdir(PLATFORMS_PATH) if os.path.isdir(os.path.join(PLATFORMS_PATH, f))]:
99
if not platform.startswith("__") and not platform.endswith("__"):
1010
# Platforms __init__.py execution
11-
init_path = f"{PLATFORMS_PATH}/{platform}/__init__.py"
12-
spec = importlib.util.spec_from_file_location("__init__", init_path)
13-
init_module = importlib.util.module_from_spec(spec)
14-
spec.loader.exec_module(init_module)
11+
execute_module(f"{PLATFORMS_PATH}/{platform}/__init__.py")
1512

1613

1714
init_platforms()
Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
UTF8_PAYLOAD_PATTERN = r"UTF8\(payload\)"
22
NUM_VALUE_PATTERN = r"(?P<num_value>\d+(?:\.\d+)*)"
3-
SINGLE_QUOTES_VALUE_PATTERN = r"""'(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'"""
3+
SINGLE_QUOTES_VALUE_PATTERN = (
4+
r"""'(?P<s_q_value>(?:[:a-zA-Zа-яА-Я\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'""" # noqa: RUF001
5+
)
46
TABLE_PATTERN = r"\s+FROM\s+[a-zA-Z.\-*]+"
57
TABLE_GROUP_PATTERN = r"\s+FROM\s+(?P<table>[a-zA-Z.\-*]+)"

0 commit comments

Comments
 (0)