S3 minio alts

minio-alternatives/apache-ozone/docker-compose.yml

@@ -0,0 +1,111 @@
+services:
+  scm:
+    image: apache/ozone:2.1.0
+    ports:
+      - "9876:9876"
+    environment:
+      OZONE-SITE.XML_ozone.scm.names: scm
+      OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/metadata
+      OZONE-SITE.XML_ozone.scm.block.client.address: scm
+      OZONE-SITE.XML_ozone.scm.client.address: scm
+      OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
+      OZONE-SITE.XML_ozone.replication: "1"
+      OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "1"
+      ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+    command: ["ozone", "scm"]
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9876/"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+
+  om:
+    image: apache/ozone:2.1.0
+    ports:
+      - "9874:9874"
+    environment:
+      OZONE-SITE.XML_ozone.om.address: om
+      OZONE-SITE.XML_ozone.om.http-address: om:9874
+      OZONE-SITE.XML_ozone.scm.names: scm
+      OZONE-SITE.XML_ozone.scm.block.client.address: scm
+      OZONE-SITE.XML_ozone.scm.client.address: scm
+      OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
+      OZONE-SITE.XML_ozone.replication: "1"
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      WAITFOR: scm:9876
+    command: ["ozone", "om"]
+    depends_on:
+      scm:
+        condition: service_healthy
+
+  datanode:
+    image: apache/ozone:2.1.0
+    ports:
+      - "9864:9864"
+    environment:
+      OZONE-SITE.XML_hdds.datanode.dir: /data/hdds
+      OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
+      OZONE-SITE.XML_ozone.scm.names: scm
+      OZONE-SITE.XML_ozone.scm.block.client.address: scm
+      OZONE-SITE.XML_ozone.scm.client.address: scm
+      OZONE-SITE.XML_ozone.replication: "1"
+    command: ["ozone", "datanode"]
+    depends_on:
+      scm:
+        condition: service_healthy
+
+  s3g:
+    image: apache/ozone:2.1.0
+    ports:
+      - "9878:9878"
+    environment:
+      OZONE-SITE.XML_ozone.om.address: om
+      OZONE-SITE.XML_ozone.scm.names: scm
+      OZONE-SITE.XML_ozone.scm.block.client.address: scm
+      OZONE-SITE.XML_ozone.scm.client.address: scm
+      OZONE-SITE.XML_ozone.replication: "1"
+    command: ["ozone", "s3g"]
+    depends_on:
+      scm:
+        condition: service_healthy
+      om:
+        condition: service_started
+      datanode:
+        condition: service_started
+
+  mc:
+    image: minio/mc:latest
+    depends_on:
+      - s3g
+    entrypoint: >
+      /bin/sh -c "
+      sleep 10;
+      until (mc alias set ozone http://s3g:9878 admin password) do echo '...waiting...' && sleep 1; done;
+      mc mb -p ozone/warehouse;
+      tail -f /dev/null
+      "
+
+  iceberg-rest:
+    image: tabulario/iceberg-rest:latest
+    depends_on:
+      - mc
+    ports:
+      - "8181:8181"
+    environment:
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: admin
+      AWS_SECRET_ACCESS_KEY: password
+      CATALOG_WAREHOUSE: s3://warehouse/
+      CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
+      CATALOG_S3_ENDPOINT: http://s3g:9878
+      CATALOG_S3_ACCESS__KEY__ID: admin
+      CATALOG_S3_SECRET__ACCESS__KEY: password
+      CATALOG_S3_PATH__STYLE__ACCESS: "true"
+      CATALOG_S3_REGION: us-east-1
+
+  duckdb:
+    image: duckdb/duckdb:latest
+    depends_on:
+      - iceberg-rest
+    stdin_open: true
+    tty: true

minio-alternatives/apache-ozone/test.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -e
+
+echo ""
+echo "    ░▒▓██████████████████████████████████████████████████▓▒░"
+echo "    ▒                                                      ▒"
+echo "    ▓      ___                    ▓▓▓▓▓                    ▓"
+echo "    █  ___( o)>  ┌───────────┐  ▓▓▓▓▓▓▓  ▒▒▒▒▒▒▒           █"
+echo "    █  \\ <_. )   │  DUCKDB   │  ▓░Local S3 ░▓  ▒▒▒▒        █"
+echo "    █   \`---'    └───────────┘   ▓▓ Ozone  ▒▒▒             █"
+echo "    ▓           ┏━━━━━━━━━━━━━┓        ░░░                 ▓"
+echo "    ▒  ≋≋≋≋≋≋≋  ┃  ICEBERG    ┃  ≋≋≋≋≋≋≋≋  Smoke test      ▒"
+echo "    ░  ≋≋≋≋≋≋≋  ┗━━━━━━━━━━━━━┛  ≋≋≋≋≋≋≋≋                  ░"
+echo "    ░▒▓██████████████████████████████████████████████████▓▒░"
+echo ""
+
+echo "1. Checking Ozone buckets (before)..."
+docker compose exec mc mc ls ozone
+echo ""
+
+echo "2. Creating Iceberg table and inserting data..."
+docker compose exec duckdb duckdb -c "
+INSTALL iceberg;
+INSTALL httpfs;
+LOAD iceberg;
+LOAD httpfs;
+
+-- Configure S3
+CREATE SECRET s3_secret (
+    TYPE S3,
+    KEY_ID 'admin',
+    SECRET 'password',
+    REGION 'us-east-1',
+    ENDPOINT 's3g:9878',
+    USE_SSL false,
+    URL_STYLE 'path'
+);
+
+-- Configure Iceberg catalog
+CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy');
+ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret);
+
+-- Create schema and table
+CREATE SCHEMA IF NOT EXISTS cat.test;
+DROP TABLE IF EXISTS cat.test.products;
+CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2));
+
+-- Insert data
+INSERT INTO cat.test.products VALUES
+    (1, 'Widget', 9.99),
+    (2, 'Gadget', 19.99),
+    (3, 'Doohickey', 14.99);
+
+-- Query data
+SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products
+UNION ALL
+SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products;
+"
+echo ""
+
+echo "3. Verifying data in DuckDB..."
+docker compose exec duckdb duckdb -c "
+INSTALL iceberg;
+INSTALL httpfs;
+LOAD iceberg;
+LOAD httpfs;
+CREATE SECRET s3_secret (TYPE S3, KEY_ID 'admin', SECRET 'password', REGION 'us-east-1', ENDPOINT 's3g:9878', USE_SSL false, URL_STYLE 'path');
+CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy');
+ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret);
+SELECT * FROM cat.test.products ORDER BY id;
+"
+echo ""
+
+echo "4. Checking Ozone bucket contents (after)..."
+docker compose exec mc mc ls --recursive ozone/warehouse/test/products/ | head -10
+echo ""

minio-alternatives/cloudserver/docker-compose.yml

@@ -0,0 +1,49 @@
+services:
+  cloudserver:
+    image: ghcr.io/scality/cloudserver:9.2.8
+    platform: linux/amd64
+    ports:
+      - "8000:8000"
+    environment:
+      S3BACKEND: mem
+      REMOTE_MANAGEMENT_DISABLE: "1"
+      LOG_LEVEL: info
+      ENDPOINT: "cloudserver,s3.docker.internal,localhost,127.0.0.1"
+      SCALITY_ACCESS_KEY_ID: admin
+      SCALITY_SECRET_ACCESS_KEY: password
+
+  mc:
+    image: minio/mc:latest
+    depends_on:
+      - cloudserver
+    entrypoint: >
+      /bin/sh -c "
+      until (mc alias set cloudserver http://cloudserver:8000 admin password) do echo '...waiting...' && sleep 1; done;
+      mc mb -p cloudserver/warehouse;
+      tail -f /dev/null
+      "
+
+  iceberg-rest:
+    image: tabulario/iceberg-rest:latest
+    depends_on:
+      - mc
+    ports:
+      - "8181:8181"
+    environment:
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: admin
+      AWS_SECRET_ACCESS_KEY: password
+      CATALOG_WAREHOUSE: s3://warehouse/
+      CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
+      CATALOG_S3_ENDPOINT: http://cloudserver:8000
+      CATALOG_S3_ACCESS__KEY__ID: admin
+      CATALOG_S3_SECRET__ACCESS__KEY: password
+      CATALOG_S3_PATH__STYLE__ACCESS: "true"
+      CATALOG_S3_REGION: us-east-1
+
+  duckdb:
+    image: duckdb/duckdb:latest
+    depends_on:
+      - iceberg-rest
+    stdin_open: true
+    tty: true

minio-alternatives/cloudserver/test.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+set -e
+
+echo ""
+echo "    ░▒▓██████████████████████████████████████████████▓▒░"
+echo "    ▒                                                      ▒"
+echo "    ▓      ___                    ▓▓▓▓▓                    ▓"
+echo "    █  ___( o)>  ┌───────────┐  ▓▓▓▓▓▓▓  ▒▒▒▒▒▒▒           █"
+echo "    █  \\ <_. )   │  DUCKDB   │  ▓░Local S3 ░▓  ▒▒▒▒        █"
+echo "    █   \`---'    └───────────┘   ▓CloudServer▒             █"
+echo "    ▓           ┏━━━━━━━━━━━━━┓        ░░░                 ▓"
+echo "    ▒  ≋≋≋≋≋≋≋  ┃  ICEBERG    ┃  ≋≋≋≋≋≋≋≋  Smoke test      ▒"
+echo "    ░  ≋≋≋≋≋≋≋  ┗━━━━━━━━━━━━━┛  ≋≋≋≋≋≋≋≋                  ░"
+echo "    ░▒▓██████████████████████████████████████████████████▓▒░"
+echo ""
+
+echo "Waiting for all services to be ready..."
+until docker compose exec mc mc ls cloudserver >/dev/null 2>&1; do
+    echo "...waiting for CloudServer..."
+    sleep 2
+done
+until curl -sf http://localhost:8181/v1/config >/dev/null 2>&1; do
+    echo "...waiting for Iceberg REST..."
+    sleep 2
+done
+echo ""
+
+echo "1. Checking CloudServer buckets (before)..."
+docker compose exec mc mc ls cloudserver
+echo ""
+
+echo "2. Creating Iceberg table and inserting data..."
+docker compose exec duckdb duckdb -c "
+INSTALL iceberg;
+INSTALL httpfs;
+LOAD iceberg;
+LOAD httpfs;
+
+-- Configure S3
+CREATE SECRET s3_secret (
+    TYPE S3,
+    KEY_ID 'admin',
+    SECRET 'password',
+    REGION 'us-east-1',
+    ENDPOINT 'cloudserver:8000',
+    USE_SSL false,
+    URL_STYLE 'path'
+);
+
+-- Configure Iceberg catalog
+CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy');
+ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret);
+
+-- Create schema and table
+CREATE SCHEMA IF NOT EXISTS cat.test;
+DROP TABLE IF EXISTS cat.test.products;
+CREATE TABLE cat.test.products (id INTEGER, name VARCHAR, price DECIMAL(10,2));
+
+-- Insert data
+INSERT INTO cat.test.products VALUES
+    (1, 'Widget', 9.99),
+    (2, 'Gadget', 19.99),
+    (3, 'Doohickey', 14.99);
+
+-- Query data
+SELECT 'Row count:' as metric, COUNT(*)::VARCHAR as value FROM cat.test.products
+UNION ALL
+SELECT 'Total value:', CAST(SUM(price) AS VARCHAR) FROM cat.test.products;
+"
+echo ""
+
+echo "3. Verifying data in DuckDB..."
+docker compose exec duckdb duckdb -c "
+INSTALL iceberg;
+INSTALL httpfs;
+LOAD iceberg;
+LOAD httpfs;
+CREATE SECRET s3_secret (TYPE S3, KEY_ID 'admin', SECRET 'password', REGION 'us-east-1', ENDPOINT 'cloudserver:8000', USE_SSL false, URL_STYLE 'path');
+CREATE SECRET iceberg_secret (TYPE iceberg, TOKEN 'dummy');
+ATTACH 'warehouse' AS cat (TYPE iceberg, ENDPOINT 'http://iceberg-rest:8181', SECRET iceberg_secret);
+SELECT * FROM cat.test.products ORDER BY id;
+"
+echo ""
+
+echo "4. Checking CloudServer bucket contents (after)..."
+docker compose exec mc mc ls --recursive cloudserver/warehouse/test/products/ | head -10
+echo ""