merge main

Author: dtsprm

uncoder-core/app/translator/core/custom_types/functions.py

@@ -15,19 +15,21 @@ class FunctionType(CustomEnum):
     latest = "latest"
 
     divide = "divide"
+    multiply = "multiply"
 
     lower = "lower"
     split = "split"
     upper = "upper"
 
     array_length = "array_length"
-    compare = "compare"
     extract_time = "extract_time"
     ipv4_is_in_range = "ipv4_is_in_range"
 
     bin = "bin"
     eval = "eval"
     fields = "fields"
+    iploc = "iploc"
+    join = "join"
     rename = "rename"
     search = "search"
     sort_limit = "sort_limit"

uncoder-core/app/translator/core/functions.py

@@ -25,8 +25,8 @@
 
 from app.translator.core.exceptions.functions import NotSupportedFunctionException
 from app.translator.core.mapping import SourceMapping
-from app.translator.core.models.field import Alias, Field
 from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
+from app.translator.core.models.query_tokens.field import Alias, Field, PredefinedField
 from app.translator.tools.utils import execute_module
 from settings import INIT_FUNCTIONS
 
@@ -83,7 +83,6 @@ def parse(self, func_body: str, raw: str) -> Function:
 class FunctionRender(ABC):
     function_names_map: ClassVar[dict[str, str]] = {}
     order_to_render: int = 0
-    in_query_render: bool = False
     render_to_prefix: bool = False
     manager: PlatformFunctionsManager = None
 
@@ -95,17 +94,19 @@ def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRe
     def render(self, function: Function, source_mapping: SourceMapping) -> str:
         raise NotImplementedError
 
-    @staticmethod
-    def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
+    def map_field(self, field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
         if isinstance(field, Alias):
             return field.name
 
-        generic_field_name = field.get_generic_field_name(source_mapping.source_id)
-        mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
-        if isinstance(mapped_field, list):
-            mapped_field = mapped_field[0]
+        if isinstance(field, Field):
+            mappings = self.manager.platform_functions.platform_query_render.mappings
+            mapped_fields = mappings.map_field(field, source_mapping)
+            return mapped_fields[0]
 
-        return mapped_field if mapped_field else field.source_name
+        if isinstance(field, PredefinedField):
+            return self.manager.platform_functions.platform_query_render.map_predefined_field(field)
+
+        raise NotSupportedFunctionException
 
 
 class PlatformFunctionsManager:
@@ -117,7 +118,6 @@ def __init__(self):
         self._parsers_map: dict[str, FunctionParser] = {}  # {platform_func_name: FunctionParser}
 
         self._renders_map: dict[str, FunctionRender] = {}  # {generic_func_name: FunctionRender}
-        self._in_query_renders_map: dict[str, FunctionRender] = {}  # {generic_func_name: FunctionRender}
         self._order_to_render: dict[str, int] = {}  # {generic_func_name: int}
 
     def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]:
@@ -126,8 +126,6 @@ def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRe
         for generic_function_name in render.function_names_map:
             self._renders_map[generic_function_name] = render
             self._order_to_render[generic_function_name] = render.order_to_render
-            if render.in_query_render:
-                self._in_query_renders_map[generic_function_name] = render
 
         return render_class
 
@@ -149,24 +147,16 @@ def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
 
         raise NotSupportedFunctionException
 
-    def get_parser(self, platform_func_name: str) -> FunctionParser:
+    def get_parser(self, platform_func_name: str) -> Optional[FunctionParser]:
         if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)):
             return parser
 
-        raise NotSupportedFunctionException
-
     def get_render(self, generic_func_name: str) -> FunctionRender:
         if INIT_FUNCTIONS and (render := self._renders_map.get(generic_func_name)):
             return render
 
         raise NotSupportedFunctionException
 
-    def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
-        if INIT_FUNCTIONS and (render := self._in_query_renders_map.get(generic_func_name)):
-            return render
-
-        raise NotSupportedFunctionException
-
     @property
     def order_to_render(self) -> dict[str, int]:
         if INIT_FUNCTIONS:

uncoder-core/app/translator/core/mapping.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Optional, TypeVar
 
 from app.translator.core.exceptions.core import StrictPlatformException
 from app.translator.core.models.platform_details import PlatformDetails

uncoder-core/app/translator/core/models/functions/base.py

@@ -1,19 +1,38 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 
-from app.translator.core.models.field import Alias, Field, FieldValue, Keyword
-from app.translator.core.models.identifier import Identifier
+from app.translator.core.models.query_tokens.field import Alias, BaseFieldsGetter, Field
+from app.translator.core.models.query_tokens.field_field import FieldField
+from app.translator.core.models.query_tokens.field_value import FieldValue
+from app.translator.core.models.query_tokens.identifier import Identifier
+from app.translator.core.models.query_tokens.keyword import Keyword
+
+if TYPE_CHECKING:
+    from app.translator.core.models.query_tokens.function_value import FunctionValue
 
 
 @dataclass
-class Function:
+class Function(BaseFieldsGetter):
     name: str = None
-    args: list[Union[Alias, Field, FieldValue, Keyword, Function, Identifier, str, bool]] = field(default_factory=list)
+    args: list[
+        Union[Alias, Field, FieldField, FieldValue, FunctionValue, Keyword, Function, Identifier, int, str, bool]
+    ] = field(default_factory=list)
     alias: Optional[Alias] = None
     raw: str = ""
 
+    @property
+    def fields(self) -> list[Field]:
+        fields = []
+        for arg in self.args:
+            if isinstance(arg, Field):
+                fields.append(arg)
+            elif isinstance(arg, (BaseFieldsGetter, Function)):
+                fields.extend(arg.fields)
+
+        return fields
+
 
 @dataclass
 class ParsedFunctions:

uncoder-core/app/translator/core/models/functions/bin.py

@@ -2,21 +2,15 @@
 from typing import Optional
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Field
+from app.translator.core.custom_types.time import TimeFrameType
 from app.translator.core.models.functions.base import Function
-from app.translator.tools.custom_enum import CustomEnum
-
-
-class SpanType(CustomEnum):
-    days = "days"
-    hours = "hours"
-    minutes = "minutes"
+from app.translator.core.models.query_tokens.field import Field
 
 
 @dataclass
 class Span:
     value: str = "1"
-    type_: str = SpanType.days
+    type_: str = TimeFrameType.days
 
 
 @dataclass
@@ -25,3 +19,7 @@ class BinFunction(Function):
     span: Optional[Span] = None
     field: Optional[Field] = None
     bins: Optional[int] = None
+
+    @property
+    def fields(self) -> list[Field]:
+        return [self.field] if self.field else []

uncoder-core/app/translator/core/models/functions/eval.py

@@ -2,9 +2,9 @@
 from typing import Union
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Alias, Field
 from app.translator.core.models.functions.base import Function
-from app.translator.core.models.identifier import Identifier
+from app.translator.core.models.query_tokens.field import Alias, Field
+from app.translator.core.models.query_tokens.identifier import Identifier
 
 
 @dataclass
@@ -17,3 +17,16 @@ class EvalArg:
 class EvalFunction(Function):
     name: str = FunctionType.eval
     args: list[EvalArg] = None
+
+    @property
+    def fields(self) -> list[Field]:
+        fields = []
+        for arg in self.args:
+            if isinstance(arg.field_, Field):
+                fields.append(arg.field_)
+            for el in arg.expression:
+                if isinstance(el, Field):
+                    fields.append(el)
+                if isinstance(el, Function):
+                    fields.extend(el.fields)
+        return fields

uncoder-core/app/translator/core/models/functions/group_by.py

@@ -1,14 +1,27 @@
-from dataclasses import Field, dataclass, field
+from dataclasses import dataclass, field
 from typing import Union
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Alias
 from app.translator.core.models.functions.base import Function
+from app.translator.core.models.query_tokens.field import Alias, Field, PredefinedField
 
 
 @dataclass
 class GroupByFunction(Function):
     name: str = FunctionType.stats
     args: list[Function] = field(default_factory=list)
-    by_clauses: list[Union[Alias, Field]] = field(default_factory=list)
+    by_clauses: list[Union[Alias, Field, PredefinedField]] = field(default_factory=list)
     filter_: Function = None
+
+    @property
+    def fields(self) -> list[Field]:
+        fields = []
+        for arg in self.args:
+            fields.extend(arg.fields)
+        for by_clause in self.by_clauses:
+            if isinstance(by_clause, Field):
+                fields.append(by_clause)
+        if self.filter_:
+            fields.extend(self.filter_.fields)
+
+        return fields

uncoder-core/app/translator/core/models/functions/join.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass, field
+from typing import Union
+
+from app.translator.core.custom_types.functions import FunctionType
+from app.translator.core.models.functions.base import Function
+from app.translator.core.models.query_container import TokenizedQueryContainer
+from app.translator.core.models.query_tokens.field import Alias, BaseFieldsGetter, Field
+from app.translator.core.models.query_tokens.field_field import FieldField
+from app.translator.core.models.query_tokens.field_value import FieldValue
+from app.translator.core.models.query_tokens.function_value import FunctionValue
+from app.translator.core.models.query_tokens.identifier import Identifier
+from app.translator.tools.custom_enum import CustomEnum
+
+
+class JoinType(CustomEnum):
+    inner = "inner"
+    left = "left"
+    right = "right"
+    cross = "cross"
+
+
+@dataclass
+class JoinFunction(Function):
+    name: str = FunctionType.join
+    alias: Alias = None
+    type_: str = JoinType.inner
+    tokenized_query_container: TokenizedQueryContainer = None
+    condition: list[Union[FieldField, FieldValue, FunctionValue, Identifier]] = field(default_factory=list)
+    preset_log_source_str: str = None
+
+    @property
+    def fields(self) -> list[Field]:
+        fields = []
+        for arg in self.condition:
+            if isinstance(arg, BaseFieldsGetter):
+                fields.extend(arg.fields)
+
+        return fields

uncoder-core/app/translator/core/models/functions/rename.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Alias, Field
 from app.translator.core.models.functions.base import Function
+from app.translator.core.models.query_tokens.field import Alias, Field
 
 
 @dataclass
@@ -15,3 +15,11 @@ class RenameArg:
 class RenameFunction(Function):
     name: str = FunctionType.rename
     args: list[RenameArg] = None
+
+    @property
+    def fields(self) -> list[Field]:
+        fields = []
+        for arg in self.args:
+            fields.append(arg.field_)
+
+        return fields

uncoder-core/app/translator/core/models/functions/sort.py

@@ -2,8 +2,8 @@
 from typing import Union
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Alias, Field
 from app.translator.core.models.functions.base import Function
+from app.translator.core.models.query_tokens.field import Alias, Field
 from app.translator.tools.custom_enum import CustomEnum
 
 
@@ -24,3 +24,15 @@ class SortLimitFunction(Function):
     name: str = FunctionType.sort_limit
     args: list[SortArg] = None
     limit: str = None
+
+    @property
+    def fields(self) -> list[Field]:
+        fields = []
+        for arg in self.args:
+            if isinstance(arg.field, Field):
+                fields.append(arg.field)
+
+            if arg.function:
+                fields.extend(arg.function.fields)
+
+        return fields