Dagster

.devcontainer/Dockerfile

@@ -8,7 +8,7 @@ RUN unzip awscliv2.zip
 RUN sudo ./aws/install
 RUN sudo apt-get install -y python3 python3-pip
 
-RUN pip install "poetry<2"
+RUN pip install "poetry>=2.0"
 ENV AWS_CONFIG_FILE=/workspaces/sds-data-manager/.aws/config
 ENV AWS_SHARED_CREDENTIALS_FILE=/workspaces/sds-data-manager/.aws/credentials
 

.devcontainer/devcontainer.json

@@ -15,5 +15,5 @@
 		}
 	},
 	"forwardPorts": [6080],
-    "postCreateCommand": "poetry install && poetry install --with layer-spice && poetry install --with layer-processing && source $(poetry env info --path)/bin/activate"
+    "postCreateCommand": "poetry install && poetry install --with layer-database && poetry install --with layer-spice && poetry install --with layer-processing && poetry install --with cdk-install"
 }

.gitignore

@@ -44,3 +44,6 @@ app*dev.py
 
 # Sphinx documentation
 docs/build/
+
+#Dagster testing
+.tmp*/

.pre-commit-config.yaml

@@ -61,3 +61,14 @@ repos:
             "--with",
             "layer-processing",
           ]
+      - id: poetry-export
+        name: export layer idex processing requirements.txt
+        args:
+          [
+            "-f",
+            "requirements.txt",
+            "-o",
+            "sds_data_manager/lambda_code/idex_processing/requirements.txt",
+            "--with",
+            "layer-idex-processing",
+          ]

Dockerfile

@@ -0,0 +1,42 @@
+FROM python:3.11-slim
+
+# System environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    POETRY_VERSION=2.4.1 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_CREATE=false \
+    DAGSTER_HOME="/opt/dagster/dagster_home" \
+    PYTHONPATH="/opt/dagster/app" \ 
+    DAGSTER_SENSOR_GRPC_TIMEOUT_SECONDS=300
+
+# Add Poetry to PATH
+ENV PATH="$POETRY_HOME/bin:$PATH"
+
+# Install system dependencies needed for Poetry and PostgreSQL
+RUN apt-get update && apt-get install -y \
+    curl \
+    build-essential \
+    libpq-dev \
+    && curl -sSL https://install.python-poetry.org | python3 - \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set the working directory
+WORKDIR /opt/dagster/app
+
+# Copy dependency management files first to leverage Docker layer caching
+COPY pyproject.toml README.md poetry.lock* ./
+
+# Copy internal project directories into the container
+COPY sds_data_manager/ ./sds_data_manager/
+
+RUN poetry install --with layer-database && poetry install --with layer-spice && poetry install --with layer-processing && poetry install --with cdk-install
+
+# Set up the Dagster home directory and copy the system configuration
+# Dagster looks for dagster.yaml in $DAGSTER_HOME
+RUN mkdir -p $DAGSTER_HOME && cp sds_data_manager/orchestration/dagster.yaml $DAGSTER_HOME/
+
+# The CMD is technically overridden by the CDK for the Webserver and Daemon tasks,
+# but providing a default makes the image easier to test locally.
+CMD ["dagster-webserver", "--read-only", "-h", "0.0.0.0", "-p", "3000", "-w", "sds_data_manager/orchestration/workspace.yaml"]

lambda_layer/database/requirements.txt

@@ -200,9 +200,9 @@ greenlet==3.5.0 ; python_version >= "3.11" and python_version < "3.13" and (plat
 idna==3.13 ; python_version >= "3.11" and python_version < "3.13" \
     --hash=sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242 \
     --hash=sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3
-imap-data-access==0.40.1 ; python_version >= "3.11" and python_version < "3.13" \
-    --hash=sha256:184c4db2d9dd7a3dbf3dbc150405a61d12f603b866e166a4ae04bd1df8aac6ab \
-    --hash=sha256:8142457ea38b577d05e48fdca31e6199222ab562229a6c0ecb1dbf2b92c84832
+imap-data-access==0.41.0 ; python_version >= "3.11" and python_version < "3.13" \
+    --hash=sha256:88c550e41c8e9fcb881787e6911bd21a5b4253be3e5ba7176a51151324588169 \
+    --hash=sha256:927966c3d0b16d9a1f8be23abdc34cff28afdec377a6e72ade7ae27496108557
 jmespath==1.1.0 ; python_version >= "3.11" and python_version < "3.13" \
     --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \
     --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64
@@ -347,9 +347,9 @@ psycopg2-binary==2.9.12 ; python_version >= "3.11" and python_version < "3.13" \
     --hash=sha256:f921f3cd87035ef7df233383011d7a53ea1d346224752c1385f1edfd790ceb6a \
     --hash=sha256:fb1828cf3da68f99e45ebce1355d65d2d12b6a78fb5dfb16247aad6bdef5f5d2 \
     --hash=sha256:ffdd7dc5463ccd61845ac37b7012d0f35a1548df9febe14f8dd549be4a0bc81e
-psycopg==3.3.3 ; python_version >= "3.11" and python_version < "3.13" \
-    --hash=sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9 \
-    --hash=sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698
+psycopg==3.3.4 ; python_version >= "3.11" and python_version < "3.13" \
+    --hash=sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a \
+    --hash=sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc
 python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "3.13" \
     --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
     --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427

lambda_layer/spice/requirements.txt

@@ -200,9 +200,9 @@ greenlet==3.5.0 ; python_version >= "3.11" and python_version < "3.13" and (plat
 idna==3.13 ; python_version >= "3.11" and python_version < "3.13" \
     --hash=sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242 \
     --hash=sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3
-imap-data-access==0.40.1 ; python_version >= "3.11" and python_version < "3.13" \
-    --hash=sha256:184c4db2d9dd7a3dbf3dbc150405a61d12f603b866e166a4ae04bd1df8aac6ab \
-    --hash=sha256:8142457ea38b577d05e48fdca31e6199222ab562229a6c0ecb1dbf2b92c84832
+imap-data-access==0.41.0 ; python_version >= "3.11" and python_version < "3.13" \
+    --hash=sha256:88c550e41c8e9fcb881787e6911bd21a5b4253be3e5ba7176a51151324588169 \
+    --hash=sha256:927966c3d0b16d9a1f8be23abdc34cff28afdec377a6e72ade7ae27496108557
 jmespath==1.1.0 ; python_version >= "3.11" and python_version < "3.13" \
     --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \
     --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64