feat(ducklake): split the catalog.db into individual files for each dataset

pysus/api/README.md

@@ -0,0 +1,2 @@
+## Roadmap
+

pysus/api/_impl/databases.py

@@ -25,7 +25,7 @@
 
 import pandas as pd
 from pysus.api.client import PySUS
-from pysus.api.types import State
+from pysus.api.types import Origin, State
 from tqdm import tqdm
 
 
@@ -519,7 +519,7 @@ def list_files(
         "CNES",
         "CIHA",
     ],
-    client: Literal["FTP", "DadosGov"] | None = None,
+    client: Origin | None = None,
     group: str | None = None,
     state: str | None = None,
     year: int | list[int] | None = None,
@@ -536,7 +536,7 @@ def list_files(
     ----------
     dataset : Literal
         Dataset name (e.g. ``"SINAN"``, ``"SINASC"``, etc.).
-    client : Literal["FTP", "DadosGov"], optional
+    client : Origin, optional
         Data source client to query.
     group : str, optional
         Group or disease code to filter by.

pysus/api/client.py

@@ -11,6 +11,8 @@
 from typing import TYPE_CHECKING, Literal
 
 import anyio
+
+from pysus.api.types import Origin
 import duckdb
 import pandas as pd
 from pysus import CACHEPATH
@@ -477,7 +479,7 @@ def get_completed_remote_paths(self) -> set[str]:
 
     async def query(
         self,
-        client: Literal["DadosGov", "FTP"] | None = None,
+        client: Origin | None = None,
         dataset: str | None = None,
         group: str | None = None,
         state: str | None = None,

pysus/api/dadosgov/client.py

@@ -11,6 +11,7 @@
 from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, PrivateAttr
 from pysus import __version__
 from pysus.api.models import BaseRemoteClient, BaseRemoteFile
+from pysus.api.types import DadosGov as DADOSGOV
 
 if TYPE_CHECKING:
     from .models import Dataset
@@ -89,7 +90,7 @@ def name(self) -> str:
         str
             The abbreviated client name ``"DadosGov"``.
         """
-        return "DadosGov"
+        return DADOSGOV
 
     @property
     def long_name(self) -> str:

pysus/api/dadosgov/models.py

@@ -30,13 +30,9 @@ def _dedup_entries(
         if m:
             stem = filename[: m.start()]
             fmt = m.group(1).lower()
-            grouped.setdefault(stem, []).append(
-                (fmt, filename, recurso, metadata)
-            )
+            grouped.setdefault(stem, []).append((fmt, filename, recurso, metadata))
         else:
-            grouped.setdefault(filename, []).append(
-                ("", filename, recurso, metadata)
-            )
+            grouped.setdefault(filename, []).append(("", filename, recurso, metadata))
 
     result: list[tuple[str, Any, dict]] = []
     for _, items in grouped.items():
@@ -249,9 +245,7 @@ class Group(BaseRemoteGroup):
     """A group of files within a dataset."""
 
     record: ConjuntoDados
-    _formatter: Callable[[str], dict[str, Any]] | None = PrivateAttr(
-        default=None
-    )
+    _formatter: Callable[[str], dict[str, Any]] | None = PrivateAttr(default=None)
 
     def __init__(
         self,
@@ -271,7 +265,8 @@ def __init__(
             A callable that extracts metadata from filenames.
         """
         super().__init__(
-            record=record, dataset=dataset  # type: ignore[call-arg]
+            record=record,
+            dataset=dataset,  # type: ignore[call-arg]
         )
         self._formatter = formatter
 
@@ -318,9 +313,7 @@ async def _fetch_files(self) -> list[BaseRemoteFile]:
         """Build File objects from the underlying resources."""
         entries: list[tuple[str, Any, dict]] = []
         for recurso in self.record.resources:
-            filename = (
-                recurso.file_name or recurso.url.split("/")[-1].split("?")[0]
-            )
+            filename = recurso.file_name or recurso.url.split("/")[-1].split("?")[0]
             if filename.lower().endswith(".pdf") or filename.startswith("get_"):
                 continue
             metadata = {}
@@ -354,7 +347,7 @@ class Dataset(BaseRemoteDataset):
     """
 
     ids: list[str] = []
-    client: "DadosGov"
+    client: DadosGov
     group_aliases: dict[str, str] = {}
 
     def __repr__(self):
@@ -369,7 +362,7 @@ def formatter(self, filename: str) -> dict[str, Any]:
     async def _fetch_content(self) -> list[Group]:
         """Fetch all groups belonging to this dataset."""
         items: list[Group] = []
-        client: "DadosGov" = self.client
+        client: DadosGov = self.client
         if self.ids:
             for group_id in self.ids:
                 record = await client.get_dataset(group_id)