Skip to content

Commit 1cf80f3

Browse files
alexvolhaalexvolha
committed
function value
1 parent 4b54f66 commit 1cf80f3

File tree

15 files changed

+245
-128
lines changed

15 files changed

+245
-128
lines changed

‎uncoder-core/app/translator/core/const.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import Union
22

33
from app.translator.core.models.field import Alias, Field, FieldValue, Keyword
4+
from app.translator.core.models.function_value import FunctionValue
45
from app.translator.core.models.identifier import Identifier
56

6-
TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias]
7+
QUERY_TOKEN_TYPE = Union[FieldValue, FunctionValue, Keyword, Identifier, Field, Alias]

‎uncoder-core/app/translator/core/models/function_value.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
from typing import Optional, Union
2+
3+
from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS
4+
from app.translator.core.models.functions.base import Function
5+
from app.translator.core.models.identifier import Identifier
6+
from app.translator.core.str_value_manager import StrValue
7+
8+
9+
class FunctionValue:
10+
def __init__(self, function: Function, operator: Identifier, value: Union[int, str, StrValue, list, tuple]):
11+
self.function = function
12+
self.operator = operator
13+
self.values = []
14+
self.__add_value(value)
15+
16+
@property
17+
def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]:
18+
if isinstance(self.values, list) and len(self.values) == 1:
19+
return self.values[0]
20+
return self.values
21+
22+
@value.setter
23+
def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None:
24+
self.values = []
25+
self.__add_value(new_value)
26+
27+
def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None:
28+
if value and isinstance(value, (list, tuple)):
29+
for v in value:
30+
self.__add_value(v)
31+
elif (
32+
value
33+
and isinstance(value, str)
34+
and value.isnumeric()
35+
and self.operator.token_type not in STR_SEARCH_OPERATORS
36+
):
37+
self.values.append(int(value))
38+
elif value is not None and isinstance(value, (int, str)):
39+
self.values.append(value)

‎uncoder-core/app/translator/core/models/query_container.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from datetime import datetime
44
from typing import Optional
55

6-
from app.translator.core.const import TOKEN_TYPE
6+
from app.translator.core.const import QUERY_TOKEN_TYPE
77
from app.translator.core.custom_types.meta_info import SeverityType
88
from app.translator.core.mapping import DEFAULT_MAPPING_NAME
99
from app.translator.core.models.field import Field
@@ -65,6 +65,6 @@ class RawQueryDictContainer:
6565

6666
@dataclass
6767
class TokenizedQueryContainer:
68-
tokens: list[TOKEN_TYPE]
68+
tokens: list[QUERY_TOKEN_TYPE]
6969
meta_info: MetaInfoContainer
7070
functions: ParsedFunctions = field(default_factory=ParsedFunctions)

‎uncoder-core/app/translator/core/parser.py‎

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@
1818

1919
import re
2020
from abc import ABC, abstractmethod
21-
from typing import Union
21+
from typing import Optional, Union
2222

23-
from app.translator.core.const import TOKEN_TYPE
23+
from app.translator.core.const import QUERY_TOKEN_TYPE
2424
from app.translator.core.exceptions.parser import TokenizerGeneralException
2525
from app.translator.core.functions import PlatformFunctions
2626
from app.translator.core.mapping import BasePlatformMappings, SourceMapping
27-
from app.translator.core.models.field import Field, FieldValue, Keyword
28-
from app.translator.core.models.functions.base import ParsedFunctions
29-
from app.translator.core.models.identifier import Identifier
27+
from app.translator.core.models.field import Field, FieldValue
28+
from app.translator.core.models.function_value import FunctionValue
29+
from app.translator.core.models.functions.base import Function
3030
from app.translator.core.models.platform_details import PlatformDetails
3131
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
3232
from app.translator.core.tokenizer import QueryTokenizer
@@ -55,24 +55,30 @@ class PlatformQueryParser(QueryParser, ABC):
5555
tokenizer: QueryTokenizer = None
5656
platform_functions: PlatformFunctions = None
5757

58-
def get_fields_tokens(self, tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[Field]:
59-
return [token.field for token in self.tokenizer.filter_tokens(tokens, FieldValue)]
60-
61-
def get_tokens_and_source_mappings(
62-
self, query: str, log_sources: dict[str, Union[str, list[str]]]
63-
) -> tuple[list[TOKEN_TYPE], list[SourceMapping]]:
58+
def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]:
6459
if not query:
6560
raise TokenizerGeneralException("Can't translate empty query. Please provide more details")
66-
tokens = self.tokenizer.tokenize(query=query)
67-
field_tokens = self.get_fields_tokens(tokens=tokens)
61+
return self.tokenizer.tokenize(query=query)
62+
63+
def get_field_tokens(
64+
self, query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
65+
) -> list[Field]:
66+
field_tokens = []
67+
for token in query_tokens:
68+
if isinstance(token, FieldValue):
69+
field_tokens.append(token.field)
70+
elif isinstance(token, FunctionValue):
71+
field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args([token.function]))
72+
73+
if functions:
74+
field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args(functions))
75+
76+
return field_tokens
77+
78+
def get_source_mappings(
79+
self, field_tokens: list[Field], log_sources: dict[str, Union[str, list[str]]]
80+
) -> list[SourceMapping]:
6881
field_names = [field.source_name for field in field_tokens]
6982
source_mappings = self.mappings.get_suitable_source_mappings(field_names=field_names, **log_sources)
7083
self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping)
71-
72-
return tokens, source_mappings
73-
74-
def set_functions_fields_generic_names(
75-
self, functions: ParsedFunctions, source_mappings: list[SourceMapping]
76-
) -> None:
77-
field_tokens = self.tokenizer.get_field_tokens_from_func_args(args=functions.functions)
78-
self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping)
84+
return source_mappings

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 40 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
from typing import ClassVar, Optional, Union
2323

2424
from app.translator.const import DEFAULT_VALUE_TYPE
25-
from app.translator.core.const import TOKEN_TYPE
25+
from app.translator.core.const import QUERY_TOKEN_TYPE
2626
from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var
2727
from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
2828
from app.translator.core.custom_types.values import ValueType
@@ -32,6 +32,7 @@
3232
from app.translator.core.functions import PlatformFunctions
3333
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
3434
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField
35+
from app.translator.core.models.function_value import FunctionValue
3536
from app.translator.core.models.functions.base import Function, RenderedFunctions
3637
from app.translator.core.models.identifier import Identifier
3738
from app.translator.core.models.platform_details import PlatformDetails
@@ -258,7 +259,9 @@ def map_predefined_field(self, predefined_field: PredefinedField) -> str:
258259

259260
return mapped_predefined_field_name
260261

261-
def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
262+
def apply_token( # noqa: PLR0911
263+
self, token: Union[FieldValue, Function, Keyword, Identifier], source_mapping: SourceMapping
264+
) -> str:
262265
if isinstance(token, FieldValue):
263266
if token.alias:
264267
mapped_fields = [token.alias.name]
@@ -286,6 +289,12 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
286289
]
287290
)
288291
return self.group_token % joined if len(cross_paired_fields) > 1 else joined
292+
if isinstance(token, FunctionValue):
293+
func_render = self.platform_functions.manager.get_in_query_render(token.function.name)
294+
rendered_func = func_render.render(token.function, source_mapping)
295+
return self.field_value_render.apply_field_value(
296+
field=rendered_func, operator=token.operator, value=token.value
297+
)
289298
if isinstance(token, Function):
290299
func_render = self.platform_functions.manager.get_in_query_render(token.name)
291300
return func_render.render(token, source_mapping)
@@ -296,7 +305,7 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
296305

297306
return token.token_type
298307

299-
def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str:
308+
def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str:
300309
result_values = []
301310
unmapped_fields = set()
302311
for token in tokens:
@@ -412,37 +421,45 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
412421
defined_raw_log_fields.append(prefix)
413422
return "\n".join(defined_raw_log_fields)
414423

424+
def _generate_from_tokenized_query_container_by_source_mapping(
425+
self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
426+
) -> str:
427+
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
428+
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
429+
430+
if source_mapping.raw_log_fields:
431+
defined_raw_log_fields = self.generate_raw_log_fields(
432+
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
433+
)
434+
prefix += f"\n{defined_raw_log_fields}"
435+
query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
436+
not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported
437+
return self.finalize_query(
438+
prefix=prefix,
439+
query=query,
440+
functions=rendered_functions.rendered,
441+
not_supported_functions=not_supported_functions,
442+
meta_info=query_container.meta_info,
443+
source_mapping=source_mapping,
444+
)
445+
415446
def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
416447
queries_map = {}
417448
errors = []
418449
source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids)
419450

420451
for source_mapping in source_mappings:
421-
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
422-
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
423452
try:
424-
if source_mapping.raw_log_fields:
425-
defined_raw_log_fields = self.generate_raw_log_fields(
426-
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
427-
)
428-
prefix += f"\n{defined_raw_log_fields}"
429-
result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
430-
except StrictPlatformException as err:
431-
errors.append(err)
432-
continue
433-
else:
434-
not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported
435-
finalized_query = self.finalize_query(
436-
prefix=prefix,
437-
query=result,
438-
functions=rendered_functions.rendered,
439-
not_supported_functions=not_supported_functions,
440-
meta_info=query_container.meta_info,
441-
source_mapping=source_mapping,
453+
finalized_query = self._generate_from_tokenized_query_container_by_source_mapping(
454+
query_container, source_mapping
442455
)
443456
if return_only_first_query_ctx_var.get() is True:
444457
return finalized_query
445458
queries_map[source_mapping.source_id] = finalized_query
459+
except StrictPlatformException as err:
460+
errors.append(err)
461+
continue
462+
446463
if not queries_map and errors:
447464
raise errors[0]
448465
return self.finalize(queries_map)

‎uncoder-core/app/translator/core/tokenizer.py‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,20 @@
2020
from abc import ABC, abstractmethod
2121
from typing import Any, ClassVar, Optional, Union
2222

23-
from app.translator.core.const import TOKEN_TYPE
23+
from app.translator.core.const import QUERY_TOKEN_TYPE
2424
from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType
2525
from app.translator.core.custom_types.values import ValueType
2626
from app.translator.core.escape_manager import EscapeManager
27+
from app.translator.core.exceptions.functions import NotSupportedFunctionException
2728
from app.translator.core.exceptions.parser import (
2829
QueryParenthesesException,
2930
TokenizerGeneralException,
3031
UnsupportedOperatorException,
3132
)
33+
from app.translator.core.functions import PlatformFunctions
3234
from app.translator.core.mapping import SourceMapping
3335
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
36+
from app.translator.core.models.function_value import FunctionValue
3437
from app.translator.core.models.functions.base import Function
3538
from app.translator.core.models.functions.eval import EvalArg
3639
from app.translator.core.models.functions.group_by import GroupByFunction
@@ -64,6 +67,7 @@ class QueryTokenizer(BaseTokenizer):
6467

6568
# do not modify, use subclasses to define this attribute
6669
field_pattern: str = None
70+
function_pattern: str = None
6771
_value_pattern: str = None
6872
value_pattern: str = None
6973
multi_value_pattern: str = None
@@ -73,6 +77,7 @@ class QueryTokenizer(BaseTokenizer):
7377
wildcard_symbol = None
7478
escape_manager: EscapeManager = None
7579
str_value_manager: StrValueManager = None
80+
platform_functions: PlatformFunctions = None
7681

7782
def __init_subclass__(cls, **kwargs):
7883
cls._validate_re_patterns()
@@ -268,9 +273,16 @@ def _check_field_value_match(self, query: str, white_space_pattern: str = r"\s+"
268273

269274
return False
270275

276+
def search_function_value(self, query: str) -> tuple[FunctionValue, str]: # noqa: ARG002
277+
raise NotSupportedFunctionException
278+
279+
@staticmethod
280+
def _check_function_value_match(query: str) -> bool: # noqa: ARG004
281+
return False
282+
271283
def _get_next_token(
272284
self, query: str
273-
) -> tuple[Union[FieldValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]:
285+
) -> tuple[Union[FieldValue, FunctionValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]:
274286
query = query.strip("\n").strip(" ").strip("\n")
275287
if query.startswith(GroupType.L_PAREN):
276288
return Identifier(token_type=GroupType.L_PAREN), query[1:]
@@ -280,6 +292,8 @@ def _get_next_token(
280292
logical_operator = logical_operator_search.group("logical_operator")
281293
pos = logical_operator_search.end()
282294
return Identifier(token_type=logical_operator.lower()), query[pos:]
295+
if self.platform_functions and self._check_function_value_match(query):
296+
return self.search_function_value(query)
283297
if self._check_field_value_match(query):
284298
return self.search_field_value(query)
285299
if self.keyword_pattern and re.match(self.keyword_pattern, query):
@@ -288,7 +302,7 @@ def _get_next_token(
288302
raise TokenizerGeneralException("Unsupported query entry")
289303

290304
@staticmethod
291-
def _validate_parentheses(tokens: list[TOKEN_TYPE]) -> None:
305+
def _validate_parentheses(tokens: list[QUERY_TOKEN_TYPE]) -> None:
292306
parentheses = []
293307
for token in tokens:
294308
if isinstance(token, Identifier) and token.token_type in (GroupType.L_PAREN, GroupType.R_PAREN):
@@ -320,8 +334,9 @@ def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]:
320334

321335
@staticmethod
322336
def filter_tokens(
323-
tokens: list[TOKEN_TYPE], token_type: Union[type[FieldValue], type[Field], type[Keyword], type[Identifier]]
324-
) -> list[TOKEN_TYPE]:
337+
tokens: list[QUERY_TOKEN_TYPE],
338+
token_type: Union[type[FieldValue], type[Field], type[Keyword], type[Identifier]],
339+
) -> list[QUERY_TOKEN_TYPE]:
325340
return [token for token in tokens if isinstance(token, token_type)]
326341

327342
def get_field_tokens_from_func_args( # noqa: PLR0912
@@ -339,6 +354,8 @@ def get_field_tokens_from_func_args( # noqa: PLR0912
339354
elif isinstance(arg, FieldValue):
340355
if arg.field:
341356
result.append(arg.field)
357+
elif isinstance(arg, FunctionValue):
358+
result.extend(self.get_field_tokens_from_func_args(args=[arg.function]))
342359
elif isinstance(arg, GroupByFunction):
343360
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
344361
result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))

‎uncoder-core/app/translator/platforms/base/aql/parsers/aql.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@
2727
from app.translator.platforms.base.aql.functions import AQLFunctions, aql_functions
2828
from app.translator.platforms.base.aql.log_source_map import LOG_SOURCE_FUNCTIONS_MAP
2929
from app.translator.platforms.base.aql.mapping import AQLMappings, aql_mappings
30-
from app.translator.platforms.base.aql.tokenizer import AQLTokenizer, aql_tokenizer
30+
from app.translator.platforms.base.aql.tokenizer import AQLTokenizer
3131
from app.translator.tools.utils import get_match_group
3232

3333

3434
class AQLQueryParser(PlatformQueryParser):
35-
tokenizer: AQLTokenizer = aql_tokenizer
35+
tokenizer: AQLTokenizer = AQLTokenizer(aql_functions)
3636
mappings: AQLMappings = aql_mappings
3737
platform_functions: AQLFunctions = aql_functions
3838

@@ -116,10 +116,10 @@ def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[
116116

117117
def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer:
118118
query, log_sources, functions = self._parse_query(raw_query_container.query)
119-
tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources)
120-
fields_tokens = self.get_fields_tokens(tokens=tokens)
121-
self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings)
119+
query_tokens = self.get_query_tokens(query)
120+
field_tokens = self.get_field_tokens(query_tokens, functions.functions)
121+
source_mappings = self.get_source_mappings(field_tokens, log_sources)
122122
meta_info = raw_query_container.meta_info
123-
meta_info.query_fields = fields_tokens
123+
meta_info.query_fields = field_tokens
124124
meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings]
125-
return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions)
125+
return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions)

0 commit comments

Comments
 (0)