Add S3 backend core

examples/s3_example.py

@@ -0,0 +1,208 @@
+"""Cachier S3 backend example.
+
+Demonstrates persistent function caching backed by AWS S3 (or any S3-compatible
+service).  Requires boto3 to be installed::
+
+    pip install cachier[s3]
+
+A real S3 bucket (or a local S3-compatible service such as MinIO / localstack)
+is needed to run this example.  Adjust the configuration variables below to
+match your environment.
+
+"""
+
+import time
+from datetime import timedelta
+
+try:
+    import boto3
+
+    from cachier import cachier
+except ImportError as exc:
+    print(f"Missing required package: {exc}")
+    print("Install with: pip install cachier[s3]")
+    raise SystemExit(1) from exc
+
+# ---------------------------------------------------------------------------
+# Configuration - adjust these to your environment
+# ---------------------------------------------------------------------------
+BUCKET_NAME = "my-cachier-bucket"
+REGION = "us-east-1"
+
+# Optional: point to a local S3-compatible service
+# ENDPOINT_URL = "http://localhost:9000"  # MinIO default
+ENDPOINT_URL = None
+
+
+# ---------------------------------------------------------------------------
+# Helper: verify S3 connectivity
+# ---------------------------------------------------------------------------
+
+
+def _check_bucket(client, bucket: str) -> bool:
+    """Return True if the bucket is accessible."""
+    try:
+        client.head_bucket(Bucket=bucket)
+        return True
+    except Exception as exc:
+        print(f"Cannot access bucket '{bucket}': {exc}")
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Demos
+# ---------------------------------------------------------------------------
+
+
+def demo_basic_caching():
+    """Show basic S3 caching: the first call computes, the second reads cache."""
+    print("\n=== Basic S3 caching ===")
+
+    @cachier(
+        backend="s3",
+        s3_bucket=BUCKET_NAME,
+        s3_region=REGION,
+        s3_endpoint_url=ENDPOINT_URL,
+    )
+    def expensive(n: int) -> int:
+        """Simulate an expensive computation."""
+        print(f"  computing expensive({n})...")
+        time.sleep(1)
+        return n * n
+
+    expensive.clear_cache()
+
+    start = time.time()
+    r1 = expensive(5)
+    t1 = time.time() - start
+    print(f"First call:  {r1}  ({t1:.2f}s)")
+
+    start = time.time()
+    r2 = expensive(5)
+    t2 = time.time() - start
+    print(f"Second call: {r2}  ({t2:.2f}s) - from cache")
+
+    assert r1 == r2
+    assert t2 < t1
+    print("Basic caching works correctly.")
+
+
+def demo_stale_after():
+    """Show stale_after: results expire and are recomputed after the timeout."""
+    print("\n=== Stale-after demo ===")
+
+    @cachier(
+        backend="s3",
+        s3_bucket=BUCKET_NAME,
+        s3_region=REGION,
+        s3_endpoint_url=ENDPOINT_URL,
+        stale_after=timedelta(seconds=3),
+    )
+    def timed(n: int) -> float:
+        print(f"  computing timed({n})...")
+        return time.time()
+
+    timed.clear_cache()
+    r1 = timed(1)
+    r2 = timed(1)
+    assert r1 == r2, "Second call should hit cache"
+
+    print("Sleeping 4 seconds so the entry becomes stale...")
+    time.sleep(4)
+
+    r3 = timed(1)
+    assert r3 > r1, "Should have recomputed after stale period"
+    print("Stale-after works correctly.")
+
+
+def demo_client_factory():
+    """Show using a callable factory instead of a pre-built client."""
+    print("\n=== Client factory demo ===")
+
+    def make_client():
+        """Lazily create a boto3 S3 client."""
+        kwargs = {"region_name": REGION}
+        if ENDPOINT_URL:
+            kwargs["endpoint_url"] = ENDPOINT_URL
+        return boto3.client("s3", **kwargs)
+
+    @cachier(
+        backend="s3",
+        s3_bucket=BUCKET_NAME,
+        s3_client_factory=make_client,
+    )
+    def compute(n: int) -> int:
+        return n + 100
+
+    compute.clear_cache()
+    assert compute(7) == compute(7)
+    print("Client factory works correctly.")
+
+
+def demo_cache_management():
+    """Show clear_cache and overwrite_cache."""
+    print("\n=== Cache management demo ===")
+    call_count = [0]
+
+    @cachier(
+        backend="s3",
+        s3_bucket=BUCKET_NAME,
+        s3_region=REGION,
+        s3_endpoint_url=ENDPOINT_URL,
+    )
+    def managed(n: int) -> int:
+        call_count[0] += 1
+        return n * 3
+
+    managed.clear_cache()
+    managed(10)
+    managed(10)
+    assert call_count[0] == 1, "Should have been called once (cached on second call)"
+
+    managed.clear_cache()
+    managed(10)
+    assert call_count[0] == 2, "Should have recomputed after cache clear"
+
+    managed(10, cachier__overwrite_cache=True)
+    assert call_count[0] == 3, "Should have recomputed due to overwrite_cache"
+    print("Cache management works correctly.")
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def main():
+    """Run all S3 backend demos."""
+    print("Cachier S3 Backend Demo")
+    print("=" * 50)
+
+    client = boto3.client(
+        "s3",
+        region_name=REGION,
+        **({"endpoint_url": ENDPOINT_URL} if ENDPOINT_URL else {}),
+    )
+
+    if not _check_bucket(client, BUCKET_NAME):
+        print(f"\nCreate the bucket first:  aws s3 mb s3://{BUCKET_NAME} --region {REGION}")
+        raise SystemExit(1)
+
+    try:
+        demo_basic_caching()
+        demo_stale_after()
+        demo_client_factory()
+        demo_cache_management()
+
+        print("\n" + "=" * 50)
+        print("All S3 demos completed successfully.")
+        print("\nKey benefits of the S3 backend:")
+        print("- Persistent cache survives process restarts")
+        print("- Shared across machines without a running service")
+        print("- Works with any S3-compatible object storage")
+    finally:
+        client.close()
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -50,6 +50,25 @@ dependencies = [
   "pympler>=1",
   "watchdog>=2.3.1",
 ]
+
+optional-dependencies.all = [
+  "boto3>=1.26",
+  "pymongo>=4",
+  "redis>=4",
+  "sqlalchemy>=2",
+]
+optional-dependencies.mongo = [
+  "pymongo>=4",
+]
+optional-dependencies.redis = [
+  "redis>=4",
+]
+optional-dependencies.s3 = [
+  "boto3>=1.26",
+]
+optional-dependencies.sql = [
+  "sqlalchemy>=2",
+]
 urls.Source = "https://github.com/python-cachier/cachier"
 # --- setuptools ---
 
@@ -177,6 +196,7 @@ markers = [
   "pickle: test the pickle core",
   "redis: test the Redis core",
   "sql: test the SQL core",
+  "s3: test the S3 core",
   "maxage: test the max_age functionality",
   "asyncio: marks tests as async",
 ]

src/cachier/_types.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Awaitable, Callable, Literal, Union
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Literal, Union
 
 if TYPE_CHECKING:
     import pymongo.collection
@@ -8,4 +8,5 @@
 HashFunc = Callable[..., str]
 Mongetter = Callable[[], Union["pymongo.collection.Collection", Awaitable["pymongo.collection.Collection"]]]
 RedisClient = Union["redis.Redis", Callable[[], Union["redis.Redis", Awaitable["redis.Redis"]]]]
-Backend = Literal["pickle", "mongo", "memory", "redis"]
+S3Client = Union[Any, Callable[[], Any]]
+Backend = Literal["pickle", "mongo", "memory", "redis", "s3"]

src/cachier/core.py

@@ -19,13 +19,14 @@
 from typing import Any, Callable, Optional, Union
 from warnings import warn
 
-from ._types import RedisClient
+from ._types import RedisClient, S3Client
 from .config import Backend, HashFunc, Mongetter, _update_with_defaults
 from .cores.base import RecalculationNeeded, _BaseCore
 from .cores.memory import _MemoryCore
 from .cores.mongo import _MongoCore
 from .cores.pickle import _PickleCore
 from .cores.redis import _RedisCore
+from .cores.s3 import _S3Core
 from .cores.sql import _SQLCore
 from .util import parse_bytes
 
@@ -170,6 +171,13 @@ def cachier(
     mongetter: Optional[Mongetter] = None,
     sql_engine: Optional[Union[str, Any, Callable[[], Any]]] = None,
     redis_client: Optional["RedisClient"] = None,
+    s3_bucket: Optional[str] = None,
+    s3_prefix: str = "cachier",
+    s3_client: Optional["S3Client"] = None,
+    s3_client_factory: Optional[Callable[[], Any]] = None,
+    s3_region: Optional[str] = None,
+    s3_endpoint_url: Optional[str] = None,
+    s3_config: Optional[Any] = None,
     stale_after: Optional[timedelta] = None,
     next_time: Optional[bool] = None,
     cache_dir: Optional[Union[str, os.PathLike]] = None,
@@ -201,9 +209,9 @@ def cachier(
         Deprecated, use :func:`~cachier.core.cachier.hash_func` instead.
     backend : str, optional
         The name of the backend to use. Valid options currently include
-        'pickle', 'mongo', 'memory', 'sql', and 'redis'. If not provided,
-        defaults to 'pickle', unless a core-associated parameter is provided
-
+        'pickle', 'mongo', 'memory', 'sql', 'redis', and 's3'. If not
+        provided, defaults to 'pickle', unless a core-associated parameter
+        is provided.
     mongetter : callable, optional
         A callable that takes no arguments and returns a pymongo.Collection
         object with writing permissions. If provided, the backend is set to
@@ -214,6 +222,20 @@ def cachier(
     redis_client : redis.Redis or callable, optional
         Redis client instance or callable returning a Redis client.
         Used for the Redis backend.
+    s3_bucket : str, optional
+        The S3 bucket name for cache storage. Required when using the S3 backend.
+    s3_prefix : str, optional
+        Key prefix applied to all S3 cache objects. Defaults to ``"cachier"``.
+    s3_client : boto3 S3 client, optional
+        A pre-configured boto3 S3 client instance.
+    s3_client_factory : callable, optional
+        A callable that returns a boto3 S3 client, allowing lazy initialization.
+    s3_region : str, optional
+        AWS region name used when auto-creating the boto3 S3 client.
+    s3_endpoint_url : str, optional
+        Custom endpoint URL for S3-compatible services such as MinIO or localstack.
+    s3_config : botocore.config.Config, optional
+        Optional botocore Config object passed when auto-creating the client.
     stale_after : datetime.timedelta, optional
         The time delta after which a cached result is considered stale. Calls
         made after the result goes stale will trigger a recalculation of the
@@ -302,6 +324,19 @@ def cachier(
             wait_for_calc_timeout=wait_for_calc_timeout,
             entry_size_limit=size_limit_bytes,
         )
+    elif backend == "s3":
+        core = _S3Core(
+            hash_func=hash_func,
+            s3_bucket=s3_bucket,
+            wait_for_calc_timeout=wait_for_calc_timeout,
+            s3_prefix=s3_prefix,
+            s3_client=s3_client,
+            s3_client_factory=s3_client_factory,
+            s3_region=s3_region,
+            s3_endpoint_url=s3_endpoint_url,
+            s3_config=s3_config,
+            entry_size_limit=size_limit_bytes,
+        )
     else:
         raise ValueError("specified an invalid core: %s" % backend)