Integrate CKB and RAG changelogs with schema updates for RAG (#422)

DSL/Liquibase/changelog/rag-search-script-v6-endpoints.sql

@@ -0,0 +1,67 @@
+-- liquibase formatted sql
+
+-- changeset Ruwini:rag-script-v6-changeset1
+CREATE TABLE public.mock_endpoints (
+    endpoint_id     UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    service_id      UUID,
+    name            VARCHAR(255) NOT NULL,
+    description     TEXT NOT NULL,
+    type            VARCHAR(50) DEFAULT 'custom_endpoint',
+    visibility      VARCHAR(20) DEFAULT 'private',
+    method          VARCHAR(10) NOT NULL,
+    url             TEXT NOT NULL,
+    params          JSONB NOT NULL DEFAULT '[]',
+    created_at      TIMESTAMP DEFAULT NOW(),
+    updated_at      TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX idx_mock_endpoints_service_id ON public.mock_endpoints(service_id);
+CREATE INDEX idx_mock_endpoints_visibility ON public.mock_endpoints(visibility);
+
+-- changeset Ruwini:rag-script-v6-changeset2
+-- Seed data: 3 test endpoints for development and testing
+
+INSERT INTO public.mock_endpoints (name, description, type, visibility, method, url, params)
+VALUES (
+    'get_public_holidays',
+    'Get public holidays for a specific country within a date range',
+    'custom_endpoint',
+    'public',
+    'GET',
+    'https://openholidaysapi.org/PublicHolidays',
+    '[
+        {"name": "countryIsoCode", "type": "string", "required": true,  "description": "ISO 3166-1 alpha-2 country code (e.g. EE for Estonia, DE for Germany)"},
+        {"name": "languageIsoCode", "type": "string", "required": false, "description": "ISO language code for the response language (e.g. EE for Estonian, EN for English)"},
+        {"name": "validFrom",       "type": "date",   "required": true,  "description": "Start date for holiday lookup in YYYY-MM-DD format"},
+        {"name": "validTo",         "type": "date",   "required": true,  "description": "End date for holiday lookup in YYYY-MM-DD format"}
+    ]'::jsonb
+);
+
+INSERT INTO public.mock_endpoints (name, description, type, visibility, method, url, params)
+VALUES (
+    'get_current_weather',
+    'Get the current weather conditions for a given city',
+    'custom_endpoint',
+    'public',
+    'GET',
+    'https://wttr.in',
+    '[
+        {"name": "city",   "type": "string", "required": true,  "description": "Name of the city to get weather for (e.g. Tallinn, London, Berlin)"},
+        {"name": "format", "type": "string", "required": false, "description": "Response format: j1 for JSON, 1 for one-line summary (default: j1)"}
+    ]'::jsonb
+);
+
+INSERT INTO public.mock_endpoints (name, description, type, visibility, method, url, params)
+VALUES (
+    'get_exchange_rate',
+    'Get the latest currency exchange rate between two currencies',
+    'custom_endpoint',
+    'public',
+    'GET',
+    'https://api.frankfurter.app/latest',
+    '[
+        {"name": "from",   "type": "string", "required": true,  "description": "The base currency code to convert from (e.g. EUR, USD, GBP)"},
+        {"name": "to",     "type": "string", "required": true,  "description": "The target currency code to convert to (e.g. USD, EUR, JPY)"},
+        {"name": "amount", "type": "number", "required": false, "description": "The amount to convert (default: 1)"}
+    ]'::jsonb
+);

DSL/Liquibase/changelog/rag-search-script-v7-schema-migration.sql

@@ -0,0 +1,57 @@
+-- liquibase formatted sql
+
+-- changeset rag-schema-migration:rag-script-v7-changeset1
+CREATE SCHEMA IF NOT EXISTS rag_search;
+GRANT USAGE ON SCHEMA rag_search TO PUBLIC;
+-- rollback DROP SCHEMA IF EXISTS rag_search;
+
+-- changeset rag-schema-migration:rag-script-v7-changeset2
+-- Move RAG module tables from public schema to rag_search schema
+
+-- v1: LLM connection and inference tables
+ALTER TABLE IF EXISTS public.llm_connections SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.inference_results SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.inference_results_references SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.llm_platforms SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.llm_models SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.embedding_platforms SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.embedding_models SET SCHEMA rag_search;
+
+-- v2: User management tables
+ALTER TABLE IF EXISTS public."user" SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.authority SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.user_authority SET SCHEMA rag_search;
+
+-- v3: Configuration table
+ALTER TABLE IF EXISTS public.configuration SET SCHEMA rag_search;
+
+-- v5: Prompt configuration table
+ALTER TABLE IF EXISTS public.prompt_configuration SET SCHEMA rag_search;
+
+-- v6: Endpoints table
+ALTER TABLE IF EXISTS public.mock_endpoints SET SCHEMA rag_search;
+
+-- v1: Agency sync and mock CKB tables
+ALTER TABLE IF EXISTS public.agency_sync SET SCHEMA rag_search;
+ALTER TABLE IF EXISTS public.mock_ckb SET SCHEMA rag_search;
+
+-- Grant permissions on all tables and sequences in rag_search schema
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA rag_search TO PUBLIC;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA rag_search TO PUBLIC;
+
+-- rollback ALTER TABLE IF EXISTS rag_search.llm_connections SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.inference_results SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.inference_results_references SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.llm_platforms SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.llm_models SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.embedding_platforms SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.embedding_models SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search."user" SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.authority SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.user_authority SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.configuration SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.prompt_configuration SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.mock_endpoints SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.agency_sync SET SCHEMA public;
+-- rollback ALTER TABLE IF EXISTS rag_search.mock_ckb SET SCHEMA public;
+-- rollback DROP SCHEMA IF EXISTS rag_search;

DSL/Liquibase/master.yml

@@ -8,4 +8,8 @@ databaseChangeLog:
   - include:
       file: changelog/rag-search-script-v4-authority-data.xml
   - include:
-      file: changelog/rag-search-script-v5-prompt-config.sql
+      file: changelog/rag-search-script-v5-prompt-config.sql
+  - include:
+      file: changelog/rag-search-script-v6-endpoints.sql
+  - include:
+      file: changelog/rag-search-script-v7-schema-migration.sql

DSL/Liquibase_production/changelog.yaml

@@ -0,0 +1,4 @@
+databaseChangeLog:
+  - includeAll:
+      path: changelog/
+      errorIfMissingOrEmpty: true

DSL/Liquibase_production/changelog/20250424191943-add-copy-row-with-modifications-function.sql

@@ -0,0 +1,82 @@
+-- liquibase formatted sql
+-- changeset Artsiom Beida:20250424191943 ignore:true
+CREATE OR REPLACE FUNCTION copy_row_with_modifications(
+    table_name_to_copy_from VARCHAR,
+    id_column_name VARCHAR,
+    id_column_conversion_expression VARCHAR,
+    id_to_copy VARCHAR,
+    modifications VARCHAR[]
+) RETURNS VARCHAR LANGUAGE plpgsql AS '
+DECLARE
+    columns VARCHAR [];
+    to_select VARCHAR [] := (ARRAY [])::VARCHAR[];
+    sql_query VARCHAR;
+    inserted_id VARCHAR;
+    modification_columns VARCHAR[] := (ARRAY [])::VARCHAR[];
+    schema_name VARCHAR;
+    table_name_only VARCHAR;
+BEGIN
+    IF position(''.'' in table_name_to_copy_from) > 0 THEN
+        schema_name := split_part(table_name_to_copy_from, ''.'', 1);
+        table_name_only := split_part(table_name_to_copy_from, ''.'', 2);
+    ELSE
+        schema_name := ''public'';
+        table_name_only := table_name_to_copy_from;
+    END IF;
+
+    SELECT ARRAY_AGG(column_name)
+    INTO columns
+    FROM information_schema.columns
+    WHERE table_schema = schema_name
+      AND table_name = table_name_only
+      AND column_name <> id_column_name;
+
+    FOR i IN 1..array_length(modifications, 1) BY 3
+    LOOP
+        modification_columns := array_append(modification_columns, modifications[i]);
+    END LOOP;
+
+    FOR i IN 1..array_length(columns, 1)
+    LOOP
+
+        IF columns[i] = ANY(modification_columns)
+        THEN
+
+            FOR j IN 1..array_length(modifications, 1) BY 3
+            LOOP
+
+                IF modifications[j] = columns[i]
+                THEN
+                    to_select := array_append(
+                        to_select,
+                        format(
+                            ''%L%s'',
+                            modifications[j + 2],
+                            modifications[j + 1]
+                        )
+                    );
+                    EXIT;
+                END IF;
+
+            END LOOP;
+
+        ELSE
+            to_select := array_append(to_select, columns[i]);
+        END IF;
+
+    END LOOP;
+
+    sql_query := format(
+        ''INSERT INTO %I.%I(%s) SELECT %s FROM %I.%I WHERE %s = %L%s RETURNING %s::VARCHAR'',
+        schema_name, table_name_only, array_to_string(columns, '', ''),
+        array_to_string(to_select, '', ''),
+        schema_name, table_name_only,
+        id_column_name, id_to_copy, id_column_conversion_expression,
+        id_column_name
+    );
+
+    EXECUTE sql_query INTO inserted_id;
+
+    RETURN inserted_id;
+END;
+';

DSL/Liquibase_production/changelog/20250424191943-add-copy-row-with-modifications-function.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd">
+    <changeSet id="20250424191943" author="Artsiom Beida">
+        <sqlFile path="changelog/20250424191943-add-copy-row-with-modifications-function.sql" />
+        <rollback>
+            <sqlFile path="changelog/20250424191943-rollback.sql" />
+        </rollback>
+    </changeSet>
+</databaseChangeLog>

DSL/Liquibase_production/changelog/20250424191943-rollback.sql

@@ -0,0 +1,3 @@
+-- liquibase formatted sql
+-- changeset Artsiom Beida:20250424191943 ignore:true
+DROP FUNCTION copy_row_with_modifications;

DSL/Liquibase_production/changelog/20250624225925-initial_schema.sql

@@ -0,0 +1,112 @@
+-- liquibase formatted sql
+-- changeset ahmer-mt:20250624225925 ignore:true
+-- Initial Migration for Data Collection System
+-- Version: 001_initial_schema
+-- Created: 2025-01-01
+
+-- Enable UUID extension
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+CREATE EXTENSION IF NOT EXISTS hstore;
+
+-- Create custom ENUM types
+CREATE TYPE agency_type AS ENUM ('client', 'api');
+CREATE TYPE source_type AS ENUM ('url_to_scrape', 'file', 'api');
+CREATE TYPE source_status_type AS ENUM ('new', 'running', 'finished', 'failed');
+CREATE TYPE source_file_status_type AS ENUM ('scraping', 'cleaning', 'finished', 'not_found', 'failed');
+CREATE TYPE source_file_type AS ENUM ('scraped_file', 'uploaded_file', 'api_file');
+
+CREATE TABLE agency (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    base_id UUID NOT NULL DEFAULT uuid_generate_v4(),
+    name TEXT NOT NULL,
+    sector TEXT,
+    external_id TEXT,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    is_deleted BOOLEAN DEFAULT FALSE,
+    zipped_data_url TEXT,
+    zip_dirty BOOLEAN DEFAULT FALSE,
+    is_zipping BOOLEAN DEFAULT FALSE,
+    type agency_type NOT NULL DEFAULT 'client',
+    data_hash TEXT
+);
+
+CREATE TABLE source (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    base_id UUID NOT NULL DEFAULT uuid_generate_v4(),
+    agency_base_id UUID NOT NULL,
+    url TEXT,
+    subsector TEXT,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    last_scraped_at TIMESTAMP WITH TIME ZONE,
+    next_scrapping_at TIMESTAMP WITH TIME ZONE DEFAULT NULL,
+    is_deleted BOOLEAN DEFAULT FALSE,
+    is_stopping BOOLEAN DEFAULT FALSE,
+    created_by TEXT,
+    type source_type NOT NULL,
+    status source_status_type NOT NULL DEFAULT 'running',
+    update_automatically BOOLEAN,
+    cron_schedule TEXT
+);
+
+
+CREATE TABLE source_run_report (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    base_id UUID NOT NULL DEFAULT uuid_generate_v4(),
+    agency_base_id UUID NOT NULL,
+    source_base_id UUID NOT NULL,
+    agency_name TEXT,
+    url TEXT,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    scraping_started_at TIMESTAMP WITH TIME ZONE,
+    scraping_finished_at TIMESTAMP WITH TIME ZONE,
+    errors INTEGER DEFAULT 0,
+    scraping_log_url TEXT,
+    cleaning_log_url TEXT,
+    is_deleted BOOLEAN DEFAULT FALSE
+);
+
+CREATE TABLE source_run_page (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    base_id UUID NOT NULL DEFAULT uuid_generate_v4(),
+    agency_base_id UUID NOT NULL,
+    source_base_id UUID NOT NULL,
+    source_run_report_base_id UUID NOT NULL,
+    url TEXT,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    scraped_at TIMESTAMP WITH TIME ZONE,
+    error_type TEXT,
+    error_message TEXT,
+    is_deleted BOOLEAN DEFAULT FALSE
+);
+
+CREATE TABLE source_file (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    base_id UUID NOT NULL DEFAULT uuid_generate_v4(),
+    source_base_id UUID NOT NULL,
+    agency_base_id UUID NOT NULL,
+    url TEXT,
+    page_title TEXT,
+    original_data_url TEXT,
+    cleaned_data_url TEXT,
+    edited_data_url TEXT,
+    original_metadata_url TEXT,
+    cleaned_metadata_url TEXT,
+    edited_metadata_url TEXT,
+    original_data_hash TEXT,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    last_scraped_at TIMESTAMP WITH TIME ZONE,
+    originally_scraped TIMESTAMP WITH TIME ZONE,
+    status source_file_status_type NOT NULL DEFAULT 'cleaning',
+    type source_file_type NOT NULL,
+    file_name TEXT,
+    external_id TEXT,
+    subsector TEXT,
+    is_excluded BOOLEAN DEFAULT FALSE,
+    is_deleted BOOLEAN DEFAULT FALSE
+);

DSL/Liquibase_production/changelog/20250624225925-initial_schema.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd">
+    <changeSet id="20250624225925" author="ahmer-mt">
+        <sqlFile path="changelog/20250624225925-initial_schema.sql" />
+        <rollback>
+            <sqlFile path="changelog/20250624225925-rollback.sql" />
+        </rollback>
+    </changeSet>
+</databaseChangeLog>

DSL/Liquibase_production/changelog/20250624225925-rollback.sql

@@ -0,0 +1,15 @@
+-- liquibase formatted sql
+-- changeset ahmer-mt:20250624225925 ignore:true
+
+DROP TABLE IF EXISTS source_file CASCADE;
+DROP TABLE IF EXISTS source_run_page CASCADE;
+DROP TABLE IF EXISTS source_run_report CASCADE;
+DROP TABLE IF EXISTS source CASCADE;
+DROP TABLE IF EXISTS agency CASCADE;
+
+-- Drop custom ENUM types
+DROP TYPE IF EXISTS source_file_type;
+DROP TYPE IF EXISTS source_file_status_type;
+DROP TYPE IF EXISTS source_status_type;
+DROP TYPE IF EXISTS source_type;
+DROP TYPE IF EXISTS agency_type;