aql, xql str value managers, func dataclasses

Author: oleksandr.volha

uncoder-core/app/translator/core/custom_types/functions.py

@@ -5,16 +5,29 @@ class FunctionType(CustomEnum):
     avg = "avg"
     count = "count"
     distinct_count = "distinct_count"
+    max = "max"
+    min = "min"
+    sum = "sum"
+
+    divide = "divide"
+
     earliest = "earliest"
+    latest = "latest"
+
+    lower = "lower"
+    upper = "upper"
+
+    compare_boolean = "compare_boolean"
+
+    ipv4_is_in_range = "ipv4_is_in_range"
+
+    bin = "bin"
     eval = "eval"
     fields = "fields"
-    latest = "latest"
-    max = "max"
-    min = "min"
     rename = "rename"
     search = "search"
-    sort = "sort"
+    sort_limit = "sort_limit"
     stats = "stats"
-    sum = "sum"
     table = "table"
+    timeframe = "timeframe"
     values = "values"

uncoder-core/app/translator/core/custom_types/tokens.py

@@ -33,6 +33,12 @@ class GroupType(CustomEnum):
 
 
 STR_SEARCH_OPERATORS = (
-    OperatorType.CONTAINS, OperatorType.NOT_CONTAINS, OperatorType.ENDSWITH, OperatorType.NOT_ENDSWITH,
-    OperatorType.STARTSWITH, OperatorType.NOT_STARTSWITH, OperatorType.REGEX, OperatorType.NOT_REGEX
+    OperatorType.CONTAINS,
+    OperatorType.NOT_CONTAINS,
+    OperatorType.ENDSWITH,
+    OperatorType.NOT_ENDSWITH,
+    OperatorType.STARTSWITH,
+    OperatorType.NOT_STARTSWITH,
+    OperatorType.REGEX,
+    OperatorType.NOT_REGEX,
 )

uncoder-core/app/translator/core/functions.py

@@ -100,7 +100,10 @@ class PlatformFunctionsManager:
     def __init__(self):
         self._parsers_map: dict[str, HigherOrderFunctionParser] = {}
         self._renders_map: dict[str, FunctionRender] = {}
+        self._in_query_renders_map: dict[str, FunctionRender] = {}
         self._names_map: dict[str, str] = {}
+        self._order_to_render: dict[str, int] = {}
+        self._render_to_prefix_functions: list[str] = []
 
     def post_init_configure(self, platform_render: PlatformQueryRender) -> None:
         raise NotImplementedError
@@ -121,6 +124,12 @@ def get_render(self, generic_func_name: str) -> FunctionRender:
 
         raise NotSupportedFunctionException
 
+    def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
+        if INIT_FUNCTIONS and (render := self._in_query_renders_map.get(generic_func_name)):
+            return render
+
+        raise NotSupportedFunctionException
+
     def get_generic_func_name(self, platform_func_name: str) -> Optional[str]:
         if INIT_FUNCTIONS and (generic_func_name := self._names_map.get(platform_func_name)):
             return generic_func_name
@@ -131,6 +140,20 @@ def get_platform_func_name(self, generic_func_name: str) -> Optional[str]:
         if INIT_FUNCTIONS:
             return self._inverted_names_map.get(generic_func_name)
 
+    @property
+    def order_to_render(self) -> dict[str, int]:
+        if INIT_FUNCTIONS:
+            return self._order_to_render
+
+        return {}
+
+    @property
+    def render_to_prefix_functions(self) -> list[str]:
+        if INIT_FUNCTIONS:
+            return self._render_to_prefix_functions
+
+        return []
+
 
 class PlatformFunctions:
     manager: PlatformFunctionsManager = PlatformFunctionsManager()
@@ -158,18 +181,27 @@ def parse(self, query: str) -> ParsedFunctions:
             invalid=invalid,
         )
 
+    def _sort_functions_to_render(self, functions: list[Function]) -> list[Function]:
+        return sorted(functions, key=lambda func: self.manager.order_to_render.get(func.name, 0))
+
     def render(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions:
         rendered = ""
+        rendered_prefix = ""
         not_supported = []
+        functions = self._sort_functions_to_render(functions)
         for func in functions:
             try:
                 func_render = self.manager.get_render(func.name)
-                rendered += self.wrap_function_with_delimiter(func_render.render(func, source_mapping))
+                _rendered = func_render.render(func, source_mapping)
+                if func.name in self.manager.render_to_prefix_functions:
+                    rendered_prefix += _rendered
+                else:
+                    rendered += self.wrap_function_with_delimiter(_rendered)
             except NotSupportedFunctionException:
                 not_supported.append(func.raw)
 
         not_supported = [self.wrap_function_with_delimiter(func.strip()) for func in not_supported]
-        return RenderedFunctions(rendered=rendered, not_supported=not_supported)
+        return RenderedFunctions(rendered_prefix=rendered_prefix, rendered=rendered, not_supported=not_supported)
 
     def wrap_function_with_delimiter(self, func: str) -> str:
         return f" {self.function_delimiter} {func}"

uncoder-core/app/translator/core/mapping.py

@@ -82,6 +82,7 @@ def __init__(
 
 class BasePlatformMappings:
     skip_load_default_mappings: bool = True
+    extend_default_mapping_with_all_fields: bool = False
 
     def __init__(self, platform_dir: str):
         self._loader = LoaderFileMappings()
@@ -116,6 +117,9 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
         if self.skip_load_default_mappings:
             source_mappings[DEFAULT_MAPPING_NAME] = default_mapping
 
+        if self.extend_default_mapping_with_all_fields:
+            source_mappings[DEFAULT_MAPPING_NAME].fields_mapping.update(default_mapping.fields_mapping)
+
         return source_mappings
 
     @staticmethod

uncoder-core/app/translator/core/models/field.py

@@ -1,11 +1,16 @@
 from typing import Optional, Union
 
-from app.translator.core.custom_types.tokens import OperatorType, STR_SEARCH_OPERATORS
+from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS, OperatorType
 from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping
 from app.translator.core.models.identifier import Identifier
 from app.translator.core.str_value_manager import StrValue
 
 
+class Alias:
+    def __init__(self, name: str):
+        self.name = name
+
+
 class Field:
     def __init__(self, source_name: str):
         self.source_name = source_name
@@ -33,8 +38,18 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
 
 
 class FieldValue:
-    def __init__(self, source_name: str, operator: Identifier, value: Union[int, str, StrValue, list, tuple]):
+    def __init__(
+        self,
+        source_name: str,
+        operator: Identifier,
+        value: Union[int, str, StrValue, list, tuple],
+        is_alias: bool = False,
+    ):
         self.field = Field(source_name=source_name)
+        self.alias = None
+        if is_alias:
+            self.alias = Alias(name=source_name)
+
         self.operator = operator
         self.values = []
         self.__add_value(value)
@@ -49,13 +64,21 @@ def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -
         if value and isinstance(value, (list, tuple)):
             for v in value:
                 self.__add_value(v)
-        elif value and isinstance(value, str) and value.isnumeric() and self.operator.token_type not in STR_SEARCH_OPERATORS:
+        elif (
+            value
+            and isinstance(value, str)
+            and value.isnumeric()
+            and self.operator.token_type not in STR_SEARCH_OPERATORS
+        ):
             self.values.append(int(value))
         elif value is not None and isinstance(value, (int, str)):
             self.values.append(value)
 
     def __repr__(self):
-        return f"{self.field.source_name} {self.operator.token_type} {self.values}"
+        if self.field:
+            return f"{self.field.source_name} {self.operator.token_type} {self.values}"
+
+        return f"{self.alias.name} {self.operator.token_type} {self.values}"
 
 
 class Keyword:

uncoder-core/app/translator/core/models/functions/base.py

@@ -1,18 +1,17 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Union
+from typing import Optional, Union
 
-from app.translator.core.models.field import Field, FieldValue, Keyword
+from app.translator.core.models.field import Alias, Field, FieldValue, Keyword
 from app.translator.core.models.identifier import Identifier
 
 
 @dataclass
 class Function:
     name: str = None
-    args: list[Union[Field, FieldValue, Keyword, Function, Identifier]] = field(default_factory=list)
-    as_clause: str = None
-    by_clauses: list[Field] = field(default_factory=list)
+    args: list[Union[Alias, Field, FieldValue, Keyword, Function, Identifier, str, bool]] = field(default_factory=list)
+    alias: Optional[Alias] = None
     raw: str = ""
 
 
@@ -21,9 +20,11 @@ class ParsedFunctions:
     functions: list[Function] = field(default_factory=list)
     not_supported: list[str] = field(default_factory=list)
     invalid: list[str] = field(default_factory=list)
+    aliases: dict[str, Function] = field(default_factory=dict)
 
 
 @dataclass
 class RenderedFunctions:
+    rendered_prefix: str = ""
     rendered: str = ""
     not_supported: list[str] = field(default_factory=list)

uncoder-core/app/translator/core/models/functions/bin.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from app.translator.core.custom_types.functions import FunctionType
+from app.translator.core.models.field import Field
+from app.translator.core.models.functions.base import Function
+from app.translator.tools.custom_enum import CustomEnum
+
+
+class SpanType(CustomEnum):
+    days = "days"
+    hours = "hours"
+    minutes = "minutes"
+
+
+@dataclass
+class Span:
+    value: str = "1"
+    type_: str = SpanType.days
+
+
+@dataclass
+class BinFunction(Function):
+    name: str = FunctionType.bin
+    span: Optional[Span] = None
+    field: Optional[Field] = None
+    bins: Optional[int] = None

uncoder-core/app/translator/core/models/functions/eval.py

@@ -2,14 +2,14 @@
 from typing import Union
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Field
+from app.translator.core.models.field import Alias, Field
 from app.translator.core.models.functions.base import Function
 from app.translator.core.models.identifier import Identifier
 
 
 @dataclass
 class EvalArg:
-    field_: Field = None
+    field_: Union[Alias, Field] = None
     expression: list[Union[Field, Function, Identifier, int, float, str]] = field(default_factory=list)
 
 

uncoder-core/app/translator/core/models/functions/group_by.py

@@ -0,0 +1,14 @@
+from dataclasses import Field, dataclass, field
+from typing import Union
+
+from app.translator.core.custom_types.functions import FunctionType
+from app.translator.core.models.field import Alias
+from app.translator.core.models.functions.base import Function
+
+
+@dataclass
+class GroupByFunction(Function):
+    name: str = FunctionType.stats
+    args: list[Function] = field(default_factory=list)
+    by_clauses: list[Union[Alias, Field]] = field(default_factory=list)
+    filter_: Function = None

uncoder-core/app/translator/core/models/functions/rename.py

@@ -1,14 +1,14 @@
 from dataclasses import dataclass
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Field
+from app.translator.core.models.field import Alias, Field
 from app.translator.core.models.functions.base import Function
 
 
 @dataclass
 class RenameArg:
     field_: Field = None
-    alias: str = None
+    alias: Alias = None
 
 
 @dataclass