Merge pull request #170 from UncoderIO/gis-8064

Gis 8064 FunctionValue class, query tokens refactoring

Author: alexvolha

uncoder-core/app/translator/core/const.py

@@ -1,6 +1,10 @@
 from typing import Union
 
-from app.translator.core.models.field import Alias, Field, FieldValue, Keyword
-from app.translator.core.models.identifier import Identifier
+from app.translator.core.models.query_tokens.field import Alias, Field
+from app.translator.core.models.query_tokens.field_field import FieldField
+from app.translator.core.models.query_tokens.field_value import FieldValue
+from app.translator.core.models.query_tokens.function_value import FunctionValue
+from app.translator.core.models.query_tokens.identifier import Identifier
+from app.translator.core.models.query_tokens.keyword import Keyword
 
-TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias]
+QUERY_TOKEN_TYPE = Union[FieldField, FieldValue, FunctionValue, Keyword, Identifier, Field, Alias]

uncoder-core/app/translator/core/custom_types/functions.py

@@ -22,7 +22,6 @@ class FunctionType(CustomEnum):
     upper = "upper"
 
     array_length = "array_length"
-    compare = "compare"
     extract_time = "extract_time"
     ipv4_is_in_range = "ipv4_is_in_range"
 

uncoder-core/app/translator/core/functions.py

@@ -25,8 +25,8 @@
 
 from app.translator.core.exceptions.functions import NotSupportedFunctionException
 from app.translator.core.mapping import SourceMapping
-from app.translator.core.models.field import Alias, Field
 from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
+from app.translator.core.models.query_tokens.field import Alias, Field
 from app.translator.tools.utils import execute_module
 from settings import INIT_FUNCTIONS
 
@@ -83,7 +83,6 @@ def parse(self, func_body: str, raw: str) -> Function:
 class FunctionRender(ABC):
     function_names_map: ClassVar[dict[str, str]] = {}
     order_to_render: int = 0
-    in_query_render: bool = False
     render_to_prefix: bool = False
     manager: PlatformFunctionsManager = None
 
@@ -117,7 +116,6 @@ def __init__(self):
         self._parsers_map: dict[str, FunctionParser] = {}  # {platform_func_name: FunctionParser}
 
         self._renders_map: dict[str, FunctionRender] = {}  # {generic_func_name: FunctionRender}
-        self._in_query_renders_map: dict[str, FunctionRender] = {}  # {generic_func_name: FunctionRender}
         self._order_to_render: dict[str, int] = {}  # {generic_func_name: int}
 
     def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]:
@@ -126,8 +124,6 @@ def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRe
         for generic_function_name in render.function_names_map:
             self._renders_map[generic_function_name] = render
             self._order_to_render[generic_function_name] = render.order_to_render
-            if render.in_query_render:
-                self._in_query_renders_map[generic_function_name] = render
 
         return render_class
 
@@ -149,24 +145,16 @@ def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
 
         raise NotSupportedFunctionException
 
-    def get_parser(self, platform_func_name: str) -> FunctionParser:
+    def get_parser(self, platform_func_name: str) -> Optional[FunctionParser]:
         if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)):
             return parser
 
-        raise NotSupportedFunctionException
-
     def get_render(self, generic_func_name: str) -> FunctionRender:
         if INIT_FUNCTIONS and (render := self._renders_map.get(generic_func_name)):
             return render
 
         raise NotSupportedFunctionException
 
-    def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
-        if INIT_FUNCTIONS and (render := self._in_query_renders_map.get(generic_func_name)):
-            return render
-
-        raise NotSupportedFunctionException
-
     @property
     def order_to_render(self) -> dict[str, int]:
         if INIT_FUNCTIONS:

uncoder-core/app/translator/core/mixins/logic.py

@@ -1,19 +1,21 @@
-from typing import Union
-
+from app.translator.core.const import QUERY_TOKEN_TYPE
 from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType
-from app.translator.core.models.field import FieldValue, Keyword
-from app.translator.core.models.identifier import Identifier
+from app.translator.core.models.query_tokens.field_field import FieldField
+from app.translator.core.models.query_tokens.field_value import FieldValue
+from app.translator.core.models.query_tokens.function_value import FunctionValue
+from app.translator.core.models.query_tokens.identifier import Identifier
+from app.translator.core.models.query_tokens.keyword import Keyword
 
 
 class ANDLogicOperatorMixin:
     @staticmethod
-    def get_missed_and_token_indices(tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[int]:
+    def get_missed_and_token_indices(tokens: list[QUERY_TOKEN_TYPE]) -> list[int]:
         missed_and_indices = []
         for index in range(len(tokens) - 1):
             token = tokens[index]
             next_token = tokens[index + 1]
             if (
-                isinstance(token, (FieldValue, Keyword))
+                isinstance(token, (FieldField, FieldValue, FunctionValue, Keyword))
                 or isinstance(token, Identifier)
                 and token.token_type == GroupType.R_PAREN
             ) and not (
@@ -23,9 +25,7 @@ def get_missed_and_token_indices(tokens: list[Union[FieldValue, Keyword, Identif
                 missed_and_indices.append(index + 1)
         return list(reversed(missed_and_indices))
 
-    def add_and_token_if_missed(
-        self, tokens: list[Union[FieldValue, Keyword, Identifier]]
-    ) -> list[Union[FieldValue, Keyword, Identifier]]:
+    def add_and_token_if_missed(self, tokens: list[QUERY_TOKEN_TYPE]) -> list[QUERY_TOKEN_TYPE]:
         indices = self.get_missed_and_token_indices(tokens=tokens)
         for index in indices:
             tokens.insert(index, Identifier(token_type=LogicalOperatorType.AND))

uncoder-core/app/translator/core/mixins/operator.py

@@ -19,7 +19,7 @@
 from typing import Optional, Union
 
 from app.translator.core.custom_types.tokens import OperatorType
-from app.translator.core.models.identifier import Identifier
+from app.translator.core.models.query_tokens.identifier import Identifier
 
 
 class WildCardMixin:

uncoder-core/app/translator/core/models/field.py

@@ -1,16 +1,24 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 
-from app.translator.core.models.field import Alias, Field, FieldValue, Keyword
-from app.translator.core.models.identifier import Identifier
+from app.translator.core.models.query_tokens.field import Alias, Field
+from app.translator.core.models.query_tokens.field_field import FieldField
+from app.translator.core.models.query_tokens.field_value import FieldValue
+from app.translator.core.models.query_tokens.identifier import Identifier
+from app.translator.core.models.query_tokens.keyword import Keyword
+
+if TYPE_CHECKING:
+    from app.translator.core.models.query_tokens.function_value import FunctionValue
 
 
 @dataclass
 class Function:
     name: str = None
-    args: list[Union[Alias, Field, FieldValue, Keyword, Function, Identifier, str, bool]] = field(default_factory=list)
+    args: list[
+        Union[Alias, Field, FieldField, FieldValue, FunctionValue, Keyword, Function, Identifier, int, str, bool]
+    ] = field(default_factory=list)
     alias: Optional[Alias] = None
     raw: str = ""
 

uncoder-core/app/translator/core/models/functions/base.py

@@ -2,21 +2,15 @@
 from typing import Optional
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Field
+from app.translator.core.custom_types.time import TimeFrameType
 from app.translator.core.models.functions.base import Function
-from app.translator.tools.custom_enum import CustomEnum
-
-
-class SpanType(CustomEnum):
-    days = "days"
-    hours = "hours"
-    minutes = "minutes"
+from app.translator.core.models.query_tokens.field import Field
 
 
 @dataclass
 class Span:
     value: str = "1"
-    type_: str = SpanType.days
+    type_: str = TimeFrameType.days
 
 
 @dataclass

uncoder-core/app/translator/core/models/functions/bin.py

@@ -2,9 +2,9 @@
 from typing import Union
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Alias, Field
 from app.translator.core.models.functions.base import Function
-from app.translator.core.models.identifier import Identifier
+from app.translator.core.models.query_tokens.field import Alias, Field
+from app.translator.core.models.query_tokens.identifier import Identifier
 
 
 @dataclass

uncoder-core/app/translator/core/models/functions/eval.py

@@ -2,8 +2,8 @@
 from typing import Union
 
 from app.translator.core.custom_types.functions import FunctionType
-from app.translator.core.models.field import Alias
 from app.translator.core.models.functions.base import Function
+from app.translator.core.models.query_tokens.field import Alias
 
 
 @dataclass