Add FS Python SDK (#195)

Signed-off-by: Zike Yang <zike@apache.org>

Author: RobertIndie

.github/workflows/ci.yaml

@@ -53,3 +53,23 @@ jobs:
       - name: Collect nohup logs
         if: failure()
         run: cat nohup.out || true
+
+  test-python-sdk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Install dependencies
+        working-directory: sdks/fs-python
+        run: |
+          pip install .
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pytest
+      - name: Run Python SDK tests
+        working-directory: sdks/fs-python
+        run: |
+          make test
+

sdks/fs-python/.gitignore

@@ -0,0 +1,4 @@
+venv
+*.egg-info
+**/__pycache__
+build

sdks/fs-python/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.13-slim
+
+WORKDIR /function
+
+# Copy the SDK files
+COPY . /function/
+
+# Install the SDK
+RUN pip install .
+
+# Set the default command
+CMD ["python", "/function/function.py"] 

sdks/fs-python/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build-image
+
+build-image:
+	docker build -t functionstream/fs-python-base .
+
+test:
+	pytest -v

sdks/fs-python/README.md

@@ -0,0 +1,194 @@
+<!--
+  Copyright 2024 Function Stream Org.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+# FunctionStream Python SDK
+
+FunctionStream SDK is a powerful Python library for building and deploying serverless functions that process messages from Apache Pulsar. It provides a simple yet flexible framework for creating event-driven applications with robust error handling, metrics collection, and resource management.
+
+## Features
+
+- **Easy Function Development**: Simple API for creating serverless functions
+- **Message Processing**: Built-in support for Apache Pulsar message processing
+- **Metrics Collection**: Automatic collection of performance metrics
+- **Resource Management**: Efficient handling of connections and resources
+- **Graceful Shutdown**: Proper cleanup of resources during shutdown
+- **Configuration Management**: Flexible configuration through YAML files
+- **Schema Validation**: Input and output schema validation
+- **Error Handling**: Comprehensive error handling and logging
+
+## Installation
+
+```bash
+pip install fs-sdk
+```
+
+## Quick Start
+
+1. Create a function that processes messages:
+
+```python
+from fs_sdk import FSFunction
+
+async def my_process_function(request_data: dict) -> dict:
+    # Process the request data
+    result = process_data(request_data)
+    return {"result": result}
+
+# Initialize and run the function
+function = FSFunction(
+    process_funcs={
+        'my_module': my_process_function
+    }
+)
+
+await function.start()
+```
+
+2. Create a configuration file (`config.yaml`):
+
+```yaml
+pulsar:
+  service_url: "pulsar://localhost:6650"
+  authPlugin: ""  # Optional
+  authParams: ""  # Optional
+
+module: "my_module"
+subscriptionName: "my-subscription"
+
+requestSource:
+  - pulsar:
+      topic: "input-topic"
+
+sink:
+  pulsar:
+    topic: "output-topic"
+```
+
+3. Define your function package (`package.yaml`):
+
+```yaml
+name: my_function
+type: pulsar
+modules:
+  my_module:
+    name: my_process
+    description: "Process incoming messages"
+    inputSchema:
+      type: object
+      properties:
+        data:
+          type: string
+      required:
+        - data
+    outputSchema:
+      type: object
+      properties:
+        result:
+          type: string
+```
+
+## Core Components
+
+### FSFunction
+
+The main class for creating serverless functions. It handles:
+- Message consumption and processing
+- Response generation
+- Resource management
+- Metrics collection
+- Error handling
+
+### Configuration
+
+The SDK uses YAML configuration files to define:
+- Pulsar connection settings
+- Module selection
+- Topic subscriptions
+- Input/output topics
+- Custom configuration parameters
+
+### Metrics
+
+Built-in metrics collection for:
+- Request processing time
+- Success/failure rates
+- Message throughput
+- Resource utilization
+
+## Examples
+
+Check out the `examples` directory for complete examples:
+
+- `string_function.py`: A simple string processing function
+- `test_string_function.py`: Test client for the string function
+- `config.yaml`: Example configuration
+- `package.yaml`: Example package definition
+
+## Best Practices
+
+1. **Error Handling**
+   - Always handle exceptions in your process functions
+   - Use proper logging for debugging
+   - Implement graceful shutdown
+
+2. **Resource Management**
+   - Close resources properly
+   - Use context managers when possible
+   - Monitor resource usage
+
+3. **Configuration**
+   - Use environment variables for sensitive data
+   - Validate configuration values
+   - Document configuration options
+
+4. **Testing**
+   - Write unit tests for your functions
+   - Test error scenarios
+   - Validate input/output schemas
+
+## Development
+
+### Prerequisites
+
+- Python 3.7+
+- Apache Pulsar
+- pip
+
+### Setup Development Environment
+
+```bash
+# Create virtual environment
+python -m venv venv
+source venv/bin/activate  # Linux/Mac
+# or
+.\venv\Scripts\activate  # Windows
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Install the package in development mode
+pip install -e .
+```
+
+### Running Tests
+
+```bash
+pytest
+```
+
+## Support
+
+For support, please open an issue in the GitHub repository or contact the maintainers. 

sdks/fs-python/examples/Dockerfile

@@ -0,0 +1,12 @@
+FROM functionstream/fs-python-base:latest
+
+WORKDIR /function
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY string_function.py .
+COPY config.yaml .
+COPY package.yaml .
+
+CMD ["python", "string_function.py"] 

sdks/fs-python/examples/Makefile

@@ -0,0 +1,5 @@
+
+build-image:
+	docker build -t my-function:latest .
+
+.DEFAULT_GOAL := build-image

sdks/fs-python/examples/config.yaml

@@ -0,0 +1,47 @@
+# Copyright 2024 Function Stream Org.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# FunctionStream Configuration File
+# This configuration file defines the settings for the string processing function example.
+
+pulsar:
+  serviceUrl: "pulsar://192.168.31.80:6650"  # Required: URL of the Pulsar broker
+  authPlugin: ""  # Optional: Authentication plugin class name
+  authParams: ""  # Optional: Authentication parameters
+
+module: "string"  # Required: Name of the module to use for processing
+
+# Optional: List of source topics to consume from
+# Note: Either sources or requestSource must be specified
+sources:
+  - pulsar:  # SourceSpec structure with pulsar configuration
+      topic: "topic-a"  # Topic name for regular message consumption
+
+# Optional: request source
+requestSource:
+  pulsar:  # SourceSpec structure with pulsar configuration
+    topic: "string-topic"  # Topic name for request messages
+
+# Required: Name of the subscription for the consumer
+subscriptionName: "test-sub"
+
+# Optional: Output sink configuration
+sink:
+  pulsar:  # SinkSpec structure with pulsar configuration
+    topic: "output"  # Topic name for output messages
+
+# Optional: Additional configuration parameters
+config:
+  - test: "Hello from config"  # Example configuration value
+  - test2: "Another config value"  # Another example configuration value

sdks/fs-python/examples/package.yaml

@@ -0,0 +1,42 @@
+# Copyright 2024 Function Stream Org.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# FunctionStream Package Configuration
+# This file defines the package metadata and function specifications for deployment.
+
+# Package name and type
+name: my_function  # Name of the function package
+type: pulsar      # Type of message broker to use
+
+# Module definitions
+modules:
+  string:  # Module name
+    name: string_process  # Function name
+    description: "Appends an exclamation mark to the input string"  # Function description
+    
+    # Input schema definition
+    inputSchema:
+      type: object
+      properties:
+        text:  # Input parameter
+          type: string  # Parameter type
+      required:
+        - text  # Required parameter
+    
+    # Output schema definition
+    outputSchema:
+      type: object
+      properties:
+        result:  # Output parameter
+          type: string  # Parameter type

sdks/fs-python/examples/requirements.txt

@@ -0,0 +1 @@
+# None. Please add your own deps here.