Feat: Replicate data from external source with a Plugin #72

plugins/data-replicator/README.md

@@ -0,0 +1,24 @@
+
+# Data Replicator Plugin
+
+This plugin replicates data from external sources to the internal SQLite database in StarbaseDB.
+
+## Features
+
+- Configurable pull interval
+- Table-specific replication
+- Append-only polling mechanism
+- Tracking of last replicated items
+
+## Configuration
+
+The plugin uses the following environment variables:
+
+- `REPLICATION_INTERVAL` - Interval in seconds for data pulling (default: 300)
+- `REPLICATION_TABLES` - Comma-separated list of tables to replicate (default: all tables)
+
+## Usage
+
+1. Configure the external database connection using the `EXTERNAL_DB_*` environment variables
+2. Set the `REPLICATION_INTERVAL` and `REPLICATION_TABLES` environment variables as needed
+3. The plugin will automatically start replicating data from the external source to the internal SQLite database

plugins/data-replicator/index.test.ts

@@ -0,0 +1,53 @@
+
+
+import { DataReplicatorPlugin } from './index'
+import { DataSource, ExternalDatabaseSource } from '../../src/types'
+
+describe('DataReplicatorPlugin', () => {
+    let plugin: DataReplicatorPlugin
+
+    beforeEach(() => {
+        plugin = new DataReplicatorPlugin()
+    })
+
+    it('should initialize with default configuration', () => {
+        expect(plugin).toBeDefined()
+    })
+
+    it('should start and stop replication', () => {
+        // Mock the data source
+        const mockDataSource = {
+            rpc: {
+                query: jest.fn().mockResolvedValue([])
+            }
+        }
+
+        // This is a simplified test - in a real test, we'd need to properly mock
+        // the plugin's internal state and methods
+        plugin['dataSource'] = mockDataSource as any
+
+        // Start replication
+        plugin.startReplication()
+        expect(plugin['isRunning']).toBe(true)
+
+        // Stop replication
+        plugin.stopReplication()
+        expect(plugin['isRunning']).toBe(false)
+    })
+
+    it('should handle different database dialects for table listing', async () => {
+        const postgresqlSource: ExternalDatabaseSource = {
+            dialect: 'postgresql',
+            host: 'localhost',
+            port: 5432,
+            user: 'user',
+            password: 'pass',
+            database: 'db',
+            defaultSchema: 'public'
+        }
+
+        const tables = await plugin['getTablesToReplicate'](postgresqlSource)
+        expect(tables).toEqual([])
+    })
+})
+