feat: log runs jsonl before db removal

docs/basic-command-line-interface-cli.md

@@ -119,7 +119,9 @@ You only need to re-add those `output.xml` files (with the correct `-z`/`--timez
 robotdashboard -r index=0,index=1:4;9,index=10  
 robotdashboard --removeruns 'run_start=2024-07-30 15:27:20.184407,index=20'  
 robotdashboard -r alias=some_cool_alias,tag=prod,tag=dev -r alias=alias12345  
-robotdashboard -r limit=10
+robotdashboard -r limit=10  
+# Log data of removed runs in jsonl  
+robotdashboard -r limit=10 --log-removed-runs "/myLogDir/removedRuns.jsonl"
 ```
 - Optional: `-r` or `--removeruns` specifies one or more runs to remove.  
 - Multiple values are separated by commas (,).  

robotframework_dashboard/arguments.py

@@ -272,6 +272,20 @@ def _parse_arguments(self):
             nargs="*",
             default=None,
         )
+        db_group.add_argument(
+            "--log-removed-runs",
+            metavar="PATH",
+            help=(
+                "Log (append) run data to jsonl before removing from the db\n"
+                "  • Supply a path to the desired .jsonl\n"
+                "Examples:\n"
+                "  • '--log-removed-runs /tmp/removed_runs.jsonl'\n"
+            ),
+            action="store",
+            type=str,
+            default=None,
+            dest="run_rm_log_path",
+        )
         db_group.add_argument(
             "-c",
             "--databaseclass",
@@ -637,5 +651,6 @@ def _process_arguments(self, arguments):
             "ssl_certfile": ssl_certfile,
             "ssl_keyfile": ssl_keyfile,
             "log_url": arguments.logurl,
+            "run_rm_log_path": arguments.run_rm_log_path,
         }
         return dotdict(provided_args)

robotframework_dashboard/database.py

@@ -5,6 +5,7 @@
 from time import time
 from datetime import datetime, timezone
 from typing import Union
+import json
 
 # Explicit adapter for datetime -> ISO string, replacing the deprecated default
 # behaviour removed in Python 3.12+. Compatible with Python 3.8+.
@@ -13,14 +14,15 @@
 
 
 class DatabaseProcessor(AbstractDatabaseProcessor):
-    def __init__(self, database_path: Path):
+    def __init__(self, database_path: Path, run_rm_log_path=None):
         """This function should handle the connection to the database
         And if required the creation of the tables"""
         self.database_path = database_path
         # handle possible subdirectories before creating database with sqlite
         path = Path(self.database_path)
         path.parent.mkdir(exist_ok=True, parents=True)
         self.connection: sqlite3.Connection
+        self.run_rm_log_path = run_rm_log_path
         # create tables if required
         self.open_database()
         self._create_tables()
@@ -383,6 +385,24 @@ def _get_run_paths(self):
             run_paths[entry["run_start"]] = entry.get("path") or ""
         return run_paths
 
+    def _get_run_data(self, run_start):
+        """Helper function to get dict of a run """
+        cursor = self.connection.cursor()
+        cursor.execute(GET_RUN_INFO_BY_RUN_START, (run_start,))
+
+        row = cursor.fetchone()
+        if not row:
+            print(f"Could not find run with run_start {run_start}")
+            return None
+
+        columns = [col[0] for col in cursor.description]
+        return dict(zip(columns, row))
+
+    def _log_run_jsonl(self, logpath, run_data):
+        """ Logs json of a run row to logpath """
+        with Path(logpath).open("a", encoding="utf-8") as f:
+            _ = f.write(json.dumps(run_data, default=str) + "\n")
+
     def remove_runs(self, remove_runs: list):
         """This function removes all provided runs and all their corresponding data"""
         run_starts, run_names, run_aliases, run_tags, _ = self._get_runs()
@@ -492,13 +512,17 @@ def _remove_by_limit(self, run: str, run_starts: list):
 
     def _remove_run(self, run_start: str):
         """Helper function to remove the data from all tables"""
-        self.connection.cursor().execute(DELETE_FROM_RUNS.format(run_start=run_start))
-        self.connection.cursor().execute(DELETE_FROM_SUITES.format(run_start=run_start))
-        self.connection.cursor().execute(DELETE_FROM_TESTS.format(run_start=run_start))
-        self.connection.cursor().execute(
-            DELETE_FROM_KEYWORDS.format(run_start=run_start)
-        )
-        self.connection.commit()
+        # grab data of the run for later to log in jsonl if flag set
+        run_data = self._get_run_data(run_start) if self.run_rm_log_path else None
+        with self.connection:
+            cursor = self.connection.cursor()
+            cursor.execute(DELETE_FROM_RUNS.format(run_start=run_start))
+            if cursor.rowcount > 0: # only cleanup related tables if a run would be deleted
+                cursor.execute(DELETE_FROM_SUITES.format(run_start=run_start))
+                cursor.execute(DELETE_FROM_TESTS.format(run_start=run_start))
+                cursor.execute(DELETE_FROM_KEYWORDS.format(run_start=run_start))
+                if self.run_rm_log_path and run_data:
+                    self._log_run_jsonl(self.run_rm_log_path, run_data)
 
     def vacuum_database(self):
         """This function vacuums the database to reduce the size after removing runs"""

robotframework_dashboard/main.py

@@ -40,6 +40,7 @@ def main():
         arguments.timezone,
         arguments.log_url,
         arguments.custom_filters,
+        arguments.run_rm_log_path,
     )
     # If arguments.start_server is provided override some required args
     if arguments.start_server:

robotframework_dashboard/queries.py

@@ -46,3 +46,5 @@
 UPDATE_RUN_PATH = """ UPDATE runs SET path="{path}" WHERE run_start="{run_start}" """
 
 VACUUM_DATABASE = """ VACUUM """
+
+GET_RUN_INFO_BY_RUN_START = """ SELECT * FROM runs WHERE run_start = ? """

robotframework_dashboard/robotdashboard.py

@@ -34,6 +34,7 @@ def __init__(
         timezone: str = "",
         log_url: Optional[str] = None,
         custom_filters: str = "",
+        run_rm_log_path: Optional[str] = None,
     ):
         """Sets the parameters provided in the command line"""
         self.database_path = database_path
@@ -57,6 +58,7 @@ def __init__(
         self.timezone = timezone
         self.log_url = log_url
         self.custom_filters = custom_filters
+        self.run_rm_log_path = run_rm_log_path
 
     def initialize_database(self, suppress=True):
         """Function that initializes the database if it does not exist
@@ -66,7 +68,7 @@ def initialize_database(self, suppress=True):
         if not suppress:
             console += self._print_console(f" 1. Database preparation")
         if not self.database_class:
-            self.database = DatabaseProcessor(self.database_path)
+            self.database = DatabaseProcessor(self.database_path, run_rm_log_path=self.run_rm_log_path)
         else:
             if not suppress:
                 console += self._print_console(
@@ -80,6 +82,13 @@ def initialize_database(self, suppress=True):
             module = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(module)
             self.database = module.DatabaseProcessor(self.database_path)
+            is_rm_log_unsupported = not hasattr(self.database, "run_rm_log_path")
+            setattr(self.database, "run_rm_log_path", self.run_rm_log_path)
+            if is_rm_log_unsupported and self.run_rm_log_path and not suppress:
+                console += self._print_console(
+                    "  WARNING The custom database class does not explicitly support '--log-removed-runs'. "
+                    "Logging might be skipped if the custom class overrides the deletion logic."
+                )
         if not suppress:
             console += self._print_console(
                 f"  created database: '{self.database_path}'"

tests/python/test_arguments.py

@@ -140,6 +140,7 @@ def _make_namespace(**kwargs):
         "ssl_keyfile": None,
         "logurl": None,
         "custom_filters": None,
+        "run_rm_log_path": None,
     }
     defaults.update(kwargs)
     return argparse.Namespace(**defaults)