[DOP-36877] Fix cleanup_partitions script

Makefile

@@ -64,7 +64,7 @@ db-cleanup-partitions: ##@DB Clean partitions
 	${PYTHON} -m data_rentgen.db.scripts.cleanup_partitions $(ARGS)
 
 db-cleanup-partitions-ci: ##@DB Clean partitions in CI
-	${PYTHON} -m data_rentgen.db.scripts.cleanup_partitions $(ARGS)
+	${COVERAGE} run -m data_rentgen.db.scripts.cleanup_partitions $(ARGS)
 
 db-views: ##@DB Create views
 	${UV} run coverage run python -m data_rentgen.db.scripts.refresh_analytic_views $(ARGS)

data_rentgen/db/scripts/cleanup_partitions.py

@@ -40,10 +40,9 @@
 ]
 
 POSTGRES_GET_PARTITIONS_QUERY = (
-    "SELECT relname as table_name "
+    "SELECT relname as table_name, relispartition as is_attached "
     "FROM pg_class "
     "WHERE relname like ANY(:table_prefixes) "
-    "AND relispartition = True "
     "AND relkind = 'r' "
     "ORDER BY relname"
 )
@@ -90,6 +89,7 @@ class TablePartition:
     name: str
     begin_date: date
     granularity: Literal["year", "month", "day"] = field(default="year")
+    is_attached: bool = True
 
     @property
     def qualified_name(self):
@@ -105,7 +105,7 @@ async def get_partitioned_tables(session: AsyncSession) -> dict[str, list[TableP
         logger.info("There is no partitioned tables")
         return {}
     # parse tabel_names
-    for (tabel_name,) in table_names:
+    for tabel_name, is_attached in table_names:
         granularity: Literal["year", "month", "day"] = "year"
         match = re.search(PARTITION_GRANULARITY_PATERN, tabel_name)
         if not match:
@@ -124,36 +124,39 @@ async def get_partitioned_tables(session: AsyncSession) -> dict[str, list[TableP
         else:
             day = 1
 
-        tables[name].append(TablePartition(name=name, begin_date=date(year, month, day), granularity=granularity))
+        tables[name].append(
+            TablePartition(
+                name=name,
+                begin_date=date(year, month, day),
+                granularity=granularity,
+                is_attached=is_attached,
+            ),
+        )
     return tables
 
 
-def get_tables_partitions(tables: dict[str, list[TablePartition]], keep_after: date) -> dict[str, list[TablePartition]]:
+def filter_partitions(tables: dict[str, list[TablePartition]], keep_after: date) -> dict[str, list[TablePartition]]:
     return {
         table_name: [partition for partition in partitions if partition.begin_date < keep_after]
         for table_name, partitions in tables.items()
     }
 
 
 def get_queries(tables: dict[str, list[TablePartition]], command: Command) -> list[str]:
-    match command:
-        case Command.DETACH:
-            query_template = "ALTER TABLE {table_name} DETACH PARTITION {qualified_name};"
-        case Command.DROP:
-            query_template = "DROP TABLE {qualified_name};"
-        case Command.TRUNCATE:
-            query_template = "TRUNCATE TABLE {qualified_name};"
-        case _:
-            return []
-
-    return [
-        query_template.format(
-            table_name=table_name,
-            qualified_name=partition.qualified_name,
-        )
-        for table_name, partitions in tables.items()
-        for partition in partitions
-    ]
+    result: list[str] = []
+    for table_name, partitions in tables.items():
+        for partition in partitions:
+            match command:
+                case Command.DETACH if partition.is_attached:
+                    query_template = f"ALTER TABLE {table_name} DETACH PARTITION {partition.qualified_name};"
+                case Command.TRUNCATE if partition.is_attached:
+                    query_template = f"TRUNCATE TABLE {partition.qualified_name};"
+                case Command.DROP:
+                    query_template = f"DROP TABLE {partition.qualified_name};"
+                case _:
+                    continue
+            result.append(query_template)
+    return result
 
 
 def print_partitions(tables: dict[str, list[TablePartition]]):
@@ -214,20 +217,20 @@ async def main(args: list[str]) -> None:
     session_factory = create_session_factory(db_settings)
     async with session_factory() as session:
         tables = await get_partitioned_tables(session)
-        tables_to_remove = get_tables_partitions(tables, keep_after)  # type: ignore[arg-type]
+        partitions = filter_partitions(tables, keep_after)  # type: ignore[arg-type]
 
         logger.info("Partitions matching --keep-after %s:", keep_after)
-        print_partitions(tables_to_remove)
+        print_partitions(partitions)
         match params.command:
             case Command.DRY_RUN:
                 pass
             case Command.DETACH:
-                await detach_partitions(tables_to_remove, session)
+                await detach_partitions(partitions, session)
             case Command.DROP:
-                await detach_partitions(tables_to_remove, session)
-                await drop_partitions(tables_to_remove, session)
+                await detach_partitions(partitions, session)
+                await drop_partitions(partitions, session)
             case Command.TRUNCATE:
-                await truncate_partitions(tables_to_remove, session)
+                await truncate_partitions(partitions, session)
             case _:
                 logger.error("No such command: %s", params.command)
 

docker-compose.yml

@@ -47,7 +47,7 @@ services:
   db-cleanup-partitions:
     image: mtsrus/data-rentgen:${VERSION:-latest}
     command: |
-      python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1year' '+%Y-%m-%d')
+      python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1 year' '+%Y-%m-%d')
     env_file: .env.docker
     depends_on:
       db-migration:

docs/changelog/next_release/449.bugfix.rst

@@ -0,0 +1 @@
+Fix ``data_rentgen.db.scripts.cleanup_partitions`` script ignored detached partitions while dropping old partitions.

docs/reference/database/cleanup_partitions_cli.rst

@@ -50,15 +50,15 @@ This command will log which partitions would be affected if you were to clean up
 
 .. code:: shell
 
-    python3 -m data_rentgen.db.scripts.cleanup_partitions detach_partitions --keep-after 2024-01-01
+    python3 -m data_rentgen.db.scripts.cleanup_partitions detach --keep-after 2024-01-01
 
 This will detach all partitions created before January 1, 2024, from their parent tables. The detached tables will still exist with their data.
 
 3. Remove Data and Drop Partitions Older Than a Specific Date:
 
 .. code:: shell
 
-    python3 -m data_rentgen.db.scripts.cleanup_partitions remove_data --keep-after 2024-01-01
+    python3 -m data_rentgen.db.scripts.cleanup_partitions drop --keep-after 2024-01-01
 
 This will detach and then **drop all partitions** created before January 1, 2024, permanently deleting their data.
 

docs/reference/database/index.rst

@@ -166,7 +166,7 @@ Without Docker
 
     # read settings from .env file, and run script using a specific venv with all required dependencies
     0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.create_partitions"
-    0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1year' '+%Y-%m-%d')"
+    0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1 year' '+%Y-%m-%d')"
     0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.refresh_analytic_views"
 
 See also

mddocs/reference/database/cleanup_partitions_cli.md

@@ -37,15 +37,15 @@ It's automatically inditifies partitioned tables and their granularity.
 2. Detach Partitions Older Than a Specific Date:
 
     ```shell
-    python3 -m data_rentgen.db.scripts.cleanup_partitions detach_partitions --keep-after 2024-01-01
+    python3 -m data_rentgen.db.scripts.cleanup_partitions detach --keep-after 2024-01-01
     ```
 
     This will detach all partitions created before January 1, 2024, from their parent tables. The detached tables will still exist with their data.
 
 3. Remove Data and Drop Partitions Older Than a Specific Date:
 
     ```shell
-    python3 -m data_rentgen.db.scripts.cleanup_partitions remove_data --keep-after 2024-01-01
+    python3 -m data_rentgen.db.scripts.cleanup_partitions drop --keep-after 2024-01-01
     ```
 
     This will detach and then **drop all partitions** created before January 1, 2024, permanently deleting their data.

mddocs/reference/database/index.md

@@ -158,7 +158,7 @@ By default, database is created with no data. To seed database with some example
   ```text
   # read settings from .env file, and run script using a specific venv with all required dependencies
   0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.create_partitions"
-  0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1year' '+%Y-%m-%d')"
+  0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1 year' '+%Y-%m-%d')"
   0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.refresh_analytic_views"
   ```