Merge pull request #54 from CivicDataLab/dev

Prod release - 12th March

Author: Deepthi-Chand

.github/workflows/publish-sdk.yml

@@ -23,6 +23,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          ref: ${{ github.ref }}
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -35,13 +36,30 @@ jobs:
           pip install build twine
           pip install -e ".[dev]"
 
-      - name: Update version in setup.py
+      - name: Update version in __version__.py
         run: |
-          sed -i "s/version=\".*\"/version=\"${{ inputs.version }}\"/" setup.py
+          echo '"""Version information for DataSpace SDK."""' > dataspace_sdk/__version__.py
+          echo "" >> dataspace_sdk/__version__.py
+          echo "__version__ = \"${{ inputs.version }}\"" >> dataspace_sdk/__version__.py
 
-      - name: Update version in pyproject.toml
+      - name: Update version in pyproject.toml (if exists)
         run: |
-          sed -i "s/version = \".*\"/version = \"${{ inputs.version }}\"/" pyproject.toml
+          if [ -f pyproject.toml ]; then
+            sed -i "s/version = \".*\"/version = \"${{ inputs.version }}\"/" pyproject.toml
+          fi
+
+      - name: Commit and push version changes
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add dataspace_sdk/__version__.py
+          if [ -f pyproject.toml ]; then git add pyproject.toml; fi
+          git commit -m "Bump SDK version to ${{ inputs.version }}" || echo "No changes to commit"
+
+          # Get current branch name
+          BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)
+          echo "Pushing to branch: $BRANCH_NAME"
+          git push origin $BRANCH_NAME || echo "No changes to push"
 
       - name: Run tests
         run: |
@@ -65,14 +83,6 @@ jobs:
         run: |
           twine upload dist/*
 
-      - name: Commit version changes
-        run: |
-          git config --local user.email "github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git add setup.py pyproject.toml
-          git commit -m "Bump version to ${{ inputs.version }}" || echo "No changes to commit"
-          git push || echo "No changes to push"
-
       - name: Create GitHub Release
         uses: actions/create-release@v1
         env:

DataSpace/settings.py

@@ -22,12 +22,15 @@
 
 from .cache_settings import *
 
-env = environ.Env(DEBUG=(bool, False))
-DEBUG = env.bool("DEBUG", default=True)
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
+
+# Load .env file FIRST, before reading any env variables
+env = environ.Env(DEBUG=(bool, False))
 environ.Env.read_env(os.path.join(BASE_DIR, ".env"))
 
+DEBUG = env.bool("DEBUG", default=True)
+
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/
 
@@ -274,6 +277,9 @@
     "search.documents.dataset_document": "dataset",
     "search.documents.usecase_document": "usecase",
     "search.documents.aimodel_document": "aimodel",
+    "search.documents.collaborative_document": "collaborative",
+    "search.documents.publisher_document.OrganizationPublisherDocument": "organization_publisher",
+    "search.documents.publisher_document.UserPublisherDocument": "user_publisher",
 }
 
 
@@ -301,9 +307,7 @@
 
 # Swagger Settings
 SWAGGER_SETTINGS = {
-    "SECURITY_DEFINITIONS": {
-        "Bearer": {"type": "apiKey", "name": "Authorization", "in": "header"}
-    }
+    "SECURITY_DEFINITIONS": {"Bearer": {"type": "apiKey", "name": "Authorization", "in": "header"}}
 }
 
 # Structured Logging Configuration
@@ -370,17 +374,11 @@
 
 # OpenTelemetry Sampling Configuration
 OTEL_TRACES_SAMPLER = "parentbased_traceidratio"
-OTEL_TRACES_SAMPLER_ARG = os.getenv(
-    "OTEL_TRACES_SAMPLER_ARG", "1.0"
-)  # Sample 100% in dev
+OTEL_TRACES_SAMPLER_ARG = os.getenv("OTEL_TRACES_SAMPLER_ARG", "1.0")  # Sample 100% in dev
 
 # OpenTelemetry Metrics Configuration
-OTEL_METRIC_EXPORT_INTERVAL_MILLIS = int(
-    os.getenv("OTEL_METRIC_EXPORT_INTERVAL_MILLIS", "30000")
-)
-OTEL_METRIC_EXPORT_TIMEOUT_MILLIS = int(
-    os.getenv("OTEL_METRIC_EXPORT_TIMEOUT_MILLIS", "30000")
-)
+OTEL_METRIC_EXPORT_INTERVAL_MILLIS = int(os.getenv("OTEL_METRIC_EXPORT_INTERVAL_MILLIS", "30000"))
+OTEL_METRIC_EXPORT_TIMEOUT_MILLIS = int(os.getenv("OTEL_METRIC_EXPORT_TIMEOUT_MILLIS", "30000"))
 
 # OpenTelemetry Instrumentation Configuration
 OTEL_PYTHON_DJANGO_INSTRUMENT = True

Dockerfile

@@ -1,12 +1,47 @@
 FROM python:3.10
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
-RUN echo 'deb http://archive.debian.org/debian stretch main contrib non-free' >> /etc/apt/sources.list && \
-    apt-get update && \
+
+# Install system dependencies
+RUN apt-get update && \
     apt-get autoremove -y && \
-    apt-get install -y libssl1.0-dev curl git nano wget && \
-    apt-get install -y gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget && \
-    rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/partial/*
+    apt-get install -y \
+        curl \
+        git \
+        nano \
+        wget \
+        ca-certificates \
+        fonts-liberation \
+        libasound2 \
+        libatk1.0-0 \
+        libcairo2 \
+        libcups2 \
+        libdbus-1-3 \
+        libexpat1 \
+        libfontconfig1 \
+        libgdk-pixbuf-2.0-0 \
+        libglib2.0-0 \
+        libgtk-3-0 \
+        libnspr4 \
+        libnss3 \
+        libpango-1.0-0 \
+        libpangocairo-1.0-0 \
+        libx11-6 \
+        libx11-xcb1 \
+        libxcb1 \
+        libxcomposite1 \
+        libxcursor1 \
+        libxdamage1 \
+        libxext6 \
+        libxfixes3 \
+        libxi6 \
+        libxrandr2 \
+        libxrender1 \
+        libxss1 \
+        libxtst6 \
+        lsb-release \
+        xdg-utils && \
+    rm -rf /var/lib/apt/lists/*
 
 
 WORKDIR /code

api/admin.py

@@ -59,6 +59,7 @@ class AIModelAdmin(admin.ModelAdmin):
     list_filter = (
         "provider",
         "model_type",
+        "domain",
         "status",
         "is_public",
         "is_active",
@@ -85,7 +86,7 @@ class AIModelAdmin(admin.ModelAdmin):
             "Schema",
             {"fields": ("input_schema", "output_schema"), "classes": ("collapse",)},
         ),
-        ("Metadata", {"fields": ("tags", "metadata"), "classes": ("collapse",)}),
+        ("Metadata", {"fields": ("tags", "domain", "metadata"), "classes": ("collapse",)}),
         ("Status & Visibility", {"fields": ("status", "is_public", "is_active")}),
         (
             "Performance Metrics",

api/middleware/rate_limit.py

@@ -1,20 +1,42 @@
 import logging
+import os
 import time
 from typing import Any, Callable, Optional, cast
 
+from django.conf import settings
 from django.core.cache import cache
 from django.http import HttpRequest, HttpResponse
 from redis.exceptions import RedisError
 
 logger = logging.getLogger(__name__)
 
 
+def get_whitelisted_ips() -> set[str]:
+    """Get the set of whitelisted IPs from settings or environment variable.
+
+    Can be configured via:
+    - RATE_LIMIT_WHITELIST_IPS in Django settings (list)
+    - RATE_LIMIT_WHITELIST_IPS environment variable (comma-separated string)
+    """
+    # First check Django settings
+    whitelist = getattr(settings, "RATE_LIMIT_WHITELIST_IPS", None)
+    if whitelist:
+        return set(whitelist)
+
+    # Fall back to environment variable
+    env_whitelist = os.environ.get("RATE_LIMIT_WHITELIST_IPS", "")
+    if env_whitelist:
+        return {ip.strip() for ip in env_whitelist.split(",") if ip.strip()}
+
+    return set()
+
+
 class HttpResponseTooManyRequests(HttpResponse):
     status_code = 429
 
 
 def rate_limit_middleware(
-    get_response: Callable[[HttpRequest], HttpResponse]
+    get_response: Callable[[HttpRequest], HttpResponse],
 ) -> Callable[[HttpRequest], HttpResponse]:
     """Rate limiting middleware that uses a simple cache-based counter."""
 
@@ -78,10 +100,18 @@ def check_rate_limit(request: HttpRequest) -> bool:
             return True  # Allow request on unexpected error
 
     def middleware(request: HttpRequest) -> HttpResponse:
+        client_ip = get_client_ip(request)
+        whitelisted_ips = get_whitelisted_ips()
+
+        # Skip rate limiting for whitelisted IPs
+        if client_ip in whitelisted_ips:
+            logger.debug(f"Skipping rate limit for whitelisted IP: {client_ip}")
+            return get_response(request)
+
         if not check_rate_limit(request):
             logger.warning(
                 f"Rate limited - Method: {request.method}, "
-                f"Path: {request.path}, IP: {get_client_ip(request)}"
+                f"Path: {request.path}, IP: {client_ip}"
             )
             return HttpResponseTooManyRequests()
 

api/models/AIModel.py

@@ -12,6 +12,7 @@
     EndpointAuthType,
     EndpointHTTPMethod,
     HFModelClass,
+    PromptDomain,
 )
 
 User = get_user_model()
@@ -82,8 +83,6 @@ class AIModel(models.Model):
         blank=True,
         related_name="ai_models",
     )
-    # API Configuration
-    # Endpoints are stored in separate ModelEndpoint table for flexibility
 
     # Model Capabilities
     supports_streaming = models.BooleanField(default=False)
@@ -92,19 +91,34 @@ class AIModel(models.Model):
     )
     supported_languages = models.JSONField(
         default=list,
+        blank=True,
+        null=True,
         help_text="List of supported language codes (e.g., ['en', 'es', 'fr'])",
     )
 
     # Input/Output Schema
-    input_schema = models.JSONField(default=dict, help_text="Expected input format and parameters")
-    output_schema = models.JSONField(default=dict, help_text="Expected output format")
+    input_schema = models.JSONField(
+        default=dict, blank=True, null=True, help_text="Expected input format and parameters"
+    )
+    output_schema = models.JSONField(
+        default=dict, blank=True, null=True, help_text="Expected output format"
+    )
 
     # Metadata
     tags = models.ManyToManyField("api.Tag", blank=True)
     sectors = models.ManyToManyField("api.Sector", blank=True, related_name="ai_models")
     geographies = models.ManyToManyField("api.Geography", blank=True, related_name="ai_models")
+    domain = models.CharField(
+        max_length=200,
+        choices=PromptDomain.choices,
+        blank=True,
+        null=True,
+        help_text="Domain or category (e.g., healthcare, education, legal)",
+    )
     metadata = models.JSONField(
         default=dict,
+        blank=True,
+        null=True,
         help_text="Additional metadata (training data info, limitations, etc.)",
     )