[DSLTradingMode] add and migrate TV trading mode for DSL

.github/workflows/main.yml

@@ -65,6 +65,7 @@ jobs:
           - packages/commons
           - packages/evaluators
           - packages/node
+          - packages/flow
           - packages/services
           - packages/tentacles_manager
           - packages/trading
@@ -101,7 +102,7 @@ jobs:
         fi
 
     - name: Install tentacles
-      if:  matrix.package == 'octobot'
+      if:  matrix.package == 'octobot' || matrix.package == 'packages/node' || matrix.package == 'packages/flow'
       run: |
         mkdir -p output
         OctoBot tentacles -d packages/tentacles -p any_platform.zip
@@ -113,11 +114,16 @@ jobs:
           pytest tests -n auto --dist loadfile
           pytest --ignore=tentacles/Trading/Exchange tentacles -n auto --dist loadfile
         else
-          cd ${{ matrix.package }}
-          if [ "${{ matrix.package }}" = "packages/tentacles_manager" ] || [ "${{ matrix.package }}" = "packages/node" ]; then
-            pytest tests
+          if [ "${{ matrix.package }}" = "packages/node" ] || [ "${{ matrix.package }}" = "packages/flow" ]; then
+            echo "Running tests from root dir to allow tentacles import"
+            PYTHONPATH=.:$PYTHONPATH pytest ${{ matrix.package }}/tests -n auto --dist loadfile
           else
-            pytest tests -n auto --dist loadfile
+            cd ${{ matrix.package }}
+            if [ "${{ matrix.package }}" = "packages/tentacles_manager" ]; then
+              pytest tests
+            else
+              pytest tests -n auto --dist loadfile
+            fi
           fi
         fi
       env:

BUILD

@@ -32,6 +32,7 @@ PACKAGE_SOURCES = [
     "packages/commons:octobot_commons",
     "packages/evaluators:octobot_evaluators",
     "packages/node:octobot_node",
+    "packages/flow:octobot_flow",
     "packages/services:octobot_services",
     "packages/tentacles_manager:octobot_tentacles_manager",
     "packages/trading:octobot_trading",

additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py

@@ -35,7 +35,7 @@
 import octobot_trading.personal_data as personal_data
 import octobot_trading.personal_data.orders as personal_data_orders
 import octobot_trading.util.test_tools.exchanges_test_tools as exchanges_test_tools
-import octobot_trading.util.test_tools.exchange_data as exchange_data_import
+import octobot_trading.exchanges.util.exchange_data as exchange_data_import
 import trading_backend.enums
 import octobot_tentacles_manager.api as tentacles_manager_api
 from additional_tests.exchanges_tests import get_authenticated_exchange_manager, NoProvidedCredentialsError

octobot/backtesting/minimal_data_importer.py

@@ -7,7 +7,7 @@
 import octobot_backtesting.importers
 import octobot_backtesting.enums
 
-import octobot_trading.util.test_tools.exchange_data as exchange_data_import
+import octobot_trading.exchanges.util.exchange_data as exchange_data_import
 
 
 class MinimalDataImporter(octobot_backtesting.importers.ExchangeDataImporter):

packages/commons/octobot_commons/asyncio_tools.py

@@ -14,8 +14,11 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
 import asyncio
+import contextlib
+import time
 import traceback
 import concurrent.futures
+import typing
 
 import octobot_commons.constants as constants
 import octobot_commons.logging as logging_util
@@ -117,6 +120,27 @@ async def gather_waiting_for_all_before_raising(*coros):
     return maybe_exceptions
 
 
+@contextlib.contextmanager
+def logged_waiter(self, name: str, sleep_time: float = 30) -> typing.Generator[None, None, None]:
+    """
+    Periodically log the time elapsed since the start of the waiter
+    """
+    async def _waiter() -> None:
+        t0 = time.time()
+        try:
+            await asyncio.sleep(sleep_time)
+            self.logger.info(f"{name} is still processing [{time.time() - t0:.2f} seconds] ...")
+        except asyncio.CancelledError:
+            pass
+    task = None
+    try:
+        task = asyncio.create_task(_waiter())
+        yield
+    finally:
+        if task is not None and not task.done():
+            task.cancel()
+
+
 class RLock(asyncio.Lock):
     """
     Async Lock implementing reentrancy

packages/commons/octobot_commons/constants.py

@@ -249,6 +249,7 @@ def parse_boolean_environment_var(env_key: str, default_value: str) -> bool:
 # DSL interpreter
 BASE_OPERATORS_LIBRARY = "base"
 CONTEXTUAL_OPERATORS_LIBRARY = "contextual"
+UNRESOLVED_PARAMETER_PLACEHOLDER = "UNRESOLVED_PARAMETER"
 
 # Logging
 EXCEPTION_DESC = "exception_desc"

packages/commons/octobot_commons/dataclasses/flexible_dataclass.py

@@ -25,7 +25,7 @@
 
 @dataclasses.dataclass
 class FlexibleDataclass:
-    _class_field_cache: typing.ClassVar[dict] = {}
+    _class_field_cache: typing.ClassVar[dict] = dataclasses.field(default={}, repr=False)
     """
     Implements from_dict which can be called to instantiate a new instance of this class from a dict. Using from_dict 
     ignores any additional key from the given dict that is not defined as a dataclass field.

packages/commons/octobot_commons/dsl_interpreter/__init__.py

@@ -24,7 +24,10 @@
     get_all_operators,
     clear_get_all_operators_cache,
 )
-from octobot_commons.dsl_interpreter.operator_parameter import OperatorParameter
+from octobot_commons.dsl_interpreter.operator_parameter import (
+    OperatorParameter,
+    UNINITIALIZED_VALUE,
+)
 from octobot_commons.dsl_interpreter.operator_docs import OperatorDocs
 from octobot_commons.dsl_interpreter.operators import (
     BinaryOperator,
@@ -34,15 +37,32 @@
     CallOperator,
     NameOperator,
     ExpressionOperator,
+    PreComputingCallOperator,
+    ReCallableOperatorMixin,
+    ReCallingOperatorResult,
+    ReCallingOperatorResultKeys,
+)
+from octobot_commons.dsl_interpreter.interpreter_dependency import (
+    InterpreterDependency,
+)
+from octobot_commons.dsl_interpreter.parameters_util import (
+    format_parameter_value,
+    resove_operator_params,
+    apply_resolved_parameter_value,
+    add_resolved_parameter_value,
+    has_unresolved_parameters,
+)
+from octobot_commons.dsl_interpreter.dsl_call_result import (
+    DSLCallResult,
 )
-from octobot_commons.dsl_interpreter.interpreter_dependency import InterpreterDependency
 
 __all__ = [
     "get_all_operators",
     "clear_get_all_operators_cache",
     "Interpreter",
     "Operator",
     "OperatorParameter",
+    "UNINITIALIZED_VALUE",
     "OperatorDocs",
     "BinaryOperator",
     "UnaryOperator",
@@ -51,5 +71,15 @@
     "CallOperator",
     "NameOperator",
     "ExpressionOperator",
+    "PreComputingCallOperator",
+    "ReCallableOperatorMixin",
     "InterpreterDependency",
+    "format_parameter_value",
+    "resove_operator_params",
+    "apply_resolved_parameter_value",
+    "add_resolved_parameter_value",
+    "DSLCallResult",
+    "has_unresolved_parameters",
+    "ReCallingOperatorResult",
+    "ReCallingOperatorResultKeys",
 ]

packages/commons/octobot_commons/dsl_interpreter/dsl_call_result.py

@@ -0,0 +1,37 @@
+#  Drakkar-Software OctoBot
+#  Copyright (c) Drakkar-Software, All rights reserved.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3.0 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library.
+
+import dataclasses
+import typing
+
+import octobot_commons.dataclasses
+
+
+@dataclasses.dataclass
+class DSLCallResult(octobot_commons.dataclasses.FlexibleDataclass):
+    """
+    Stores a DSL call result alongside its statement (and error if any)
+    """
+    statement: str
+    result: typing.Optional[typing.Any] = None
+    error: typing.Optional[str] = None
+
+    def succeeded(self) -> bool:
+        """
+        Check if the DSL call succeeded
+        :return: True if the DSL call succeeded, False otherwise
+        """
+        return self.error is None

packages/commons/octobot_commons/dsl_interpreter/interpreter.py

@@ -19,6 +19,8 @@
 import octobot_commons.errors
 import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator
 import octobot_commons.dsl_interpreter.interpreter_dependency as dsl_interpreter_dependency
+import octobot_commons.dsl_interpreter.parameters_util as parameters_util
+import octobot_commons.dsl_interpreter.dsl_call_result as dsl_call_result
 
 
 class Interpreter:
@@ -45,6 +47,7 @@ def __init__(
             dsl_interpreter_operator.Operator,
             dsl_interpreter_operator.ComputedOperatorParameterType,
         ] = None
+        self._parsed_expression: typing.Optional[str] = None
 
     def extend(
         self, operators: typing.List[typing.Type[dsl_interpreter_operator.Operator]]
@@ -73,7 +76,7 @@ async def interprete(
 
     def get_dependencies(
         self,
-    ) -> typing.List[dsl_interpreter_dependency.InterpreterDependency]:
+    ) -> list[dsl_interpreter_dependency.InterpreterDependency]:
         """
         Get the dependencies of the interpreter's parsed expression.
         """
@@ -109,10 +112,17 @@ def _parse_expression(self, expression: str):
         # it consists of a single expression, or 'single' if it consists of a single
         # interactive statement.
         # docs: https://docs.python.org/3/library/functions.html#compile
-        tree = ast.parse(expression, mode="eval")
-
-        # Visit the AST and convert nodes to Operator instances
-        self._operator_tree_or_constant = self._visit_node(tree.body)
+        self._parsed_expression = expression
+        try:
+            tree = ast.parse(expression, mode="eval")
+            self._operator_tree_or_constant = self._visit_node(tree.body)
+        except SyntaxError:
+            tree = ast.parse(expression, mode="single")
+            if len(tree.body) != 1:
+                raise octobot_commons.errors.DSLInterpreterError(
+                    "Single statement required when using statement mode"
+                )
+            self._operator_tree_or_constant = self._visit_node(tree.body[0])
 
     async def compute_expression(
         self,
@@ -129,6 +139,25 @@ async def compute_expression(
             return self._operator_tree_or_constant.compute()
         return self._operator_tree_or_constant
 
+    async def compute_expression_with_result(
+        self,
+    ) -> dsl_call_result.DSLCallResult:
+        """
+        Compute the result of the expression stored in self._operator_tree_or_constant.
+        If the expression is a constant, return it directly.
+        If the expression is an operator, pre_compute and compute its result.
+        """
+        try:
+            return dsl_call_result.DSLCallResult(
+                statement=self._parsed_expression,
+                result=await self.compute_expression(),
+            )
+        except octobot_commons.errors.ErrorStatementEncountered as err:
+            return dsl_call_result.DSLCallResult(
+                statement=self._parsed_expression,
+                error=err.args[0] if err.args else ""
+            )
+
     def _visit_node(self, node: typing.Optional[ast.AST]) -> typing.Union[
         dsl_interpreter_operator.Operator,
         dsl_interpreter_operator.ComputedOperatorParameterType,
@@ -159,7 +188,26 @@ def _visit_node(self, node: typing.Optional[ast.AST]) -> typing.Union[
                     )
                     for arg in node.args
                 ]
-                return operator_class(*args)
+                kwargs = {}
+                for kw in node.keywords:
+                    value = (
+                        self._get_value_from_constant_node(kw.value)
+                        if isinstance(kw.value, ast.Constant)
+                        else self._visit_node(kw.value)
+                    )
+                    if kw.arg is not None:
+                        kwargs[kw.arg] = value
+                    else:
+                        if isinstance(value, dict):
+                            kwargs.update(value)
+                        else:
+                            raise octobot_commons.errors.UnsupportedOperatorError(
+                                f"**kwargs must unpack a dict, got {type(value).__name__}"
+                            )
+                args, kwargs = parameters_util.resolve_operator_args_and_kwargs(
+                    operator_class, args, kwargs
+                )
+                return operator_class(*args, **kwargs)
             raise octobot_commons.errors.UnsupportedOperatorError(
                 f"Unknown operator: {func_name}"
             )
@@ -259,6 +307,23 @@ def _visit_node(self, node: typing.Optional[ast.AST]) -> typing.Union[
                 operands = [self._visit_node(operand) for operand in node.elts]
                 return operator_class(*operands)
 
+        if isinstance(node, ast.Dict):
+            # Dict: {"a": 1, "b": 2} or {"a": 1, **other}
+            op_name = ast.Dict.__name__
+            result = {}
+            for key, value in zip(node.keys, node.values):
+                if key is not None:
+                    result[self._visit_node(key)] = self._visit_node(value)
+                else:
+                    unpacked = self._visit_node(value)
+                    if isinstance(unpacked, dict):
+                        result.update(unpacked)
+                    else:
+                        raise octobot_commons.errors.UnsupportedOperatorError(
+                            f"** unpacking in dict requires a dict, got {type(unpacked).__name__}"
+                        )
+            return result
+
         if isinstance(node, ast.Slice):
             # Slice: slice(1, 2, 3)
             op_name = ast.Slice.__name__
@@ -269,6 +334,32 @@ def _visit_node(self, node: typing.Optional[ast.AST]) -> typing.Union[
                 step = self._visit_node(node.step)
                 return operator_class(lower, upper, step)
 
+        if isinstance(node, ast.Raise):
+            # Raise statement: raise exc [from cause] - maps to RaiseOperator
+            op_name = "raise"
+            if op_name in self.operators_by_name:
+                operator_class = self.operators_by_name[op_name]
+                args = []
+                if node.exc is not None:
+                    args.append(
+                        self._get_value_from_constant_node(node.exc)
+                        if isinstance(node.exc, ast.Constant)
+                        else self._visit_node(node.exc)
+                    )
+                if node.cause is not None:
+                    args.append(
+                        self._get_value_from_constant_node(node.cause)
+                        if isinstance(node.cause, ast.Constant)
+                        else self._visit_node(node.cause)
+                    )
+                args, kwargs = parameters_util.resolve_operator_args_and_kwargs(
+                    operator_class, args, {}
+                )
+                return operator_class(*args, **kwargs)
+            raise octobot_commons.errors.UnsupportedOperatorError(
+                f"Unknown operator: {op_name}"
+            )
+
         raise octobot_commons.errors.UnsupportedOperatorError(
             f"Unsupported AST node type: {type(node).__name__}"
         )
@@ -289,7 +380,7 @@ def _get_value_from_constant_node(
         """Extract a literal value from an AST constant node."""
         value = node.value
         # Filter out unsupported types like complex numbers or Ellipsis
-        if isinstance(value, (str, int, float, bool, type(None))):
+        if isinstance(value, (str, int, float, bool, type(None), dict)):
             return value
         raise octobot_commons.errors.UnsupportedOperatorError(
             f"Unsupported constant type: {type(value).__name__}"

packages/commons/octobot_commons/dsl_interpreter/interpreter_dependency.py

@@ -1,4 +1,3 @@
-# pylint: disable=too-many-branches,too-many-return-statements
 #  Drakkar-Software OctoBot-Commons
 #  Copyright (c) Drakkar-Software, All rights reserved.
 #