Get update from wip into llm/api-tool-calling

DSL/Liquibase/changelog/rag-search-script-v1-llm-connections.sql

@@ -124,19 +124,12 @@ CREATE INDEX idx_llm_models_platform_id ON llm_models(platform_id);
 CREATE INDEX idx_embedding_models_platform_id ON embedding_models(platform_id);
 
 CREATE TABLE public.agency_sync (
-    agency_id         VARCHAR(50) PRIMARY KEY,
+    id                VARCHAR(50) PRIMARY KEY,
     agency_data_hash  VARCHAR(255),
     data_url          TEXT,
     created_at        TIMESTAMP NOT NULL DEFAULT NOW(),
     updated_at        TIMESTAMP NOT NULL DEFAULT NOW()
 );
 
-INSERT INTO public.agency_sync (agency_id, created_at) VALUES 
-('AGENCY001', NOW());
-
-CREATE TABLE public.mock_ckb (
-    client_id         VARCHAR(50) PRIMARY KEY,
-    client_data_hash  VARCHAR(255) NOT NULL,
-    signed_s3_url     TEXT NOT NULL,
-    created_at        TIMESTAMP NOT NULL DEFAULT NOW()
-);
+INSERT INTO public.agency_sync (id, created_at) VALUES 
+('1', NOW());

DSL/Resql/rag-search/POST/get-agency-id.sql

@@ -1,4 +1,4 @@
 SELECT 
-    agency_id,
+    id,
     agency_data_hash
 FROM public.agency_sync;

DSL/Resql/rag-search/POST/mock-get-data-from-kb.sql

@@ -1,5 +1,5 @@
-SELECT 
-    client_id,
-    client_data_hash,
-    signed_s3_url
-FROM public.mock_ckb;
+-- SELECT 
+--     client_id,
+--     client_data_hash,
+--     signed_s3_url
+-- FROM public.mock_ckb;

DSL/Resql/rag-search/POST/update-agency-hash.sql

@@ -3,9 +3,9 @@ SET
     agency_data_hash = :newAgencyDataHash,
     data_url = :dataUrl,
     updated_at = NOW()
-WHERE agency_id = :agencyId
+WHERE id = :id
 RETURNING 
-    agency_id,
+    id,
     agency_data_hash,
     data_url,
     updated_at;

DSL/Ruuter.private/rag-search/POST/ckb/agency_data_import.yml

@@ -1,33 +1,33 @@
-declaration:
-  call: declare
-  version: 0.1
-  description: "Get agency data information by agency IDs"
-  method: post
-  accepts: json
-  returns: json
-  namespace: rag-search
-  allowlist:
-    body:
-      - field: agencyIds
-        type: array
-        description: "Array of unique institution IDs"
+# declaration:
+#   call: declare
+#   version: 0.1
+#   description: "Get agency data information by agency IDs"
+#   method: post
+#   accepts: json
+#   returns: json
+#   namespace: rag-search
+#   allowlist:
+#     body:
+#       - field: agencyIds
+#         type: array
+#         description: "Array of unique institution IDs"
 
-extractRequestData:
-  assign:
-    agencyIds: ${incoming.body.agencyIds || []}
-  log: "Received request for agency data: ${agencyIds}"
+# extractRequestData:
+#   assign:
+#     agencyIds: ${incoming.body.agencyIds || []}
+#   log: "Received request for agency data: ${agencyIds}"
 
-get_agency_data:
-  call: http.post
-  args:
-    url: "[#GLOBAL_CLASSIFIER_RESQL]/mock-get-data-from-kb"
-    headers:
-      type: json
-    body:
-      agencyIds: ${agencyIds}
-  result: agency_data_info
-  next: return_result
+# get_agency_data:
+#   call: http.post
+#   args:
+#     url: "[#GLOBAL_CLASSIFIER_RESQL]/mock-get-data-from-kb"
+#     headers:
+#       type: json
+#     body:
+#       agencyIds: ${agencyIds}
+#   result: agency_data_info
+#   next: return_result
 
-return_result:
-  return: ${agency_data_info.response.body}
-  next: end
+# return_result:
+#   return: ${agency_data_info.response.body}
+#   next: end

DSL/Ruuter.public/rag-search/POST/ckb/agency-data-import.yml

@@ -1,32 +1,32 @@
 declaration:
   call: declare
   version: 0.1
-  description: "Get agency data information by agency IDs"
+  description: "Get agency data from CKB import endpoint"
   method: post
   accepts: json
   returns: json
   namespace: rag-search
-  allowlist:
-    body:
-      - field: agencyIds
-        type: array
-        description: "Array of unique institution IDs"
-
-extractRequestData:
-  assign:
-    agencyIds: ${incoming.body.agencyIds || []}
-  log: "Received request for agency data: ${agencyIds}"
 
 get_agency_data:
-  call: http.post
+  call: http.get
   args:
-    url: "[#RAG_SEARCH_RESQL]/mock-get-data-from-kb"
-    headers:
-      type: json
-    body:
-      agencyIds: ${agencyIds}
+    url: "[#CKB_RUUTER_INTERNAL]/client/data/import"
   result: agency_data_info
   next: return_result
+  on_error: handle_ckb_error
+
+handle_ckb_error:
+  assign:
+    error_response:
+      success: false
+      message: "CKB data import failed"
+  log: "ERROR: Failed to fetch agency data from CKB - ${agency_data_info.error.message}"
+  next: return_error
+
+return_error:
+  status: 500
+  return: ${error_response}
+  next: end
 
 return_result:
   return: ${agency_data_info.response.body}

DSL/Ruuter.public/rag-search/POST/data/update.yml

@@ -15,12 +15,12 @@ get_agency_id:
   next: log_result
 
 log_result:
-  log: ${get_agency_id_result.response.body[0].agencyId}
+  log: ${get_agency_id_result.response.body[0].id}
   next: extract_params
 
 extract_params:
   assign:
-    single_agency_id: ${get_agency_id_result.response.body[0].agencyId}
+    single_agency_id: ${get_agency_id_result.response.body[0].id}
     agency_ids: 
       - ${single_agency_id}
     current_data_hash: ${get_agency_id_result.response.body[0].agencyDataHash}
@@ -34,34 +34,65 @@ import_agency_data:
   call: http.post
   args:
     url: "[#RAG_SEARCH_RUUTER_PUBLIC]/ckb/agency-data-import"
-    body:
-      agencyIds: ${agency_ids}
   result: importResult
   next: log_import_agency_data_response
+  on_error: handle_import_error
+
+handle_import_error:
+  log: "ERROR: Failed to import agency data - ${importResult.error.message}"
+  assign:
+    format_res: {
+      message: "Failed to fetch data from CKB - data synchronization failed",
+      operationSuccessful: false,
+      error: "CKB_IMPORT_FAILED"
+    }
+  next: return_bad_request
 
 log_import_agency_data_response:
   log: ${JSON.stringify(importResult.response)}
+  next: check_ckb_response
+
+check_ckb_response:
+  switch:
+    - condition: ${importResult.response.body == null || importResult.response.body.response == null || importResult.response.body.response == "Internal Server Error" || importResult.response.body.response.response == null}
+      next: return_ckb_error_response
+    - condition: ${importResult.response.body.response.response.length === 0}
+      next: return_no_ckb_data
   next: assign_import_agency_data
 
+return_ckb_error_response:
+  assign:
+    format_res:
+      message: "CKB service returned an error - data synchronization aborted"
+      operationSuccessful: false
+      error: "CKB_ERROR"
+  next: return_bad_request
+
+return_no_ckb_data:
+  assign:
+    error_response:
+      success: false
+      message: "Data synchronization failed - CKB agency data not found"
+  next: return_ckb_data_error
+
 assign_import_agency_data:
   assign:
-    ckb_data_hash: ${importResult.response.body.response[0].clientDataHash}
-    signed_s3_url: ${importResult.response.body.response[0].signedS3Url}
+    ckb_data_hash: ${importResult.response.body.response.response[0].agencyDataHash}
+    signed_s3_url: ${importResult.response.body.response.response[0].signedS3Url}
   next: check_has_match
 
 check_has_match:
   switch:
-    - condition: ${current_data_hash === importResult.response.body.response[0].clientDataHash}
+    - condition: ${current_data_hash !== null && current_data_hash === importResult.response.body.response.response[0].agencyDataHash}
       next: noAgencyData
-    - condition: true
-      next: sync_current_hash_with_ckb_latest_hash
+  next: sync_current_hash_with_ckb_latest_hash
 
 sync_current_hash_with_ckb_latest_hash:
   call: http.post
   args:
     url: "[#RAG_SEARCH_RESQL]/update-agency-hash"
     body:
-      agencyId: ${single_agency_id}
+      id: ${single_agency_id}
       newAgencyDataHash: ${ckb_data_hash}
       dataUrl: ${signed_s3_url}
   result: sync_agency_hash_result
@@ -105,10 +136,9 @@ assign_success_response:
 
 assign_fail_response:
   assign:
-    format_res: {
-      message: "Data synchronization failed",
-      operationSuccessful: false,
-    }
+    format_res: 
+      message: "Data synchronization failed"
+      operationSuccessful: false
   next: return_bad_request
 
 return_ok:
@@ -132,4 +162,8 @@ return_no_sync_needed:
   status: 200
   return: ${response_data}
   next: end
-
+
+return_ckb_data_error:
+  status: 404
+  return: ${error_response}
+  next: end

constants.ini

@@ -13,4 +13,5 @@ DB_PASSWORD=dbadmin
 RAG_SEARCH_RUUTER_PUBLIC_INTERNAL_SERVICE=http://ruuter:8086/services
 SERVICE_DMAPPER_HBS=http://data-mapper:3000/hbs/rag-search
 SERVICE_PROJECT_LAYER=services
-RAG_SEARCH_LLM_SERVICE=http://llm-orchestration-service:8100
+RAG_SEARCH_LLM_SERVICE=http://llm-orchestration-service:8100
+CKB_RUUTER_INTERNAL=http://ruuter-internal:8089/ckb