Features/refine chat window

src/lib/common/Breadcrumb.svelte

@@ -2,7 +2,7 @@
 	import { Row, Col, Breadcrumb, BreadcrumbItem } from '@sveltestrap/sveltestrap';
 	import Link from 'svelte-link';
 
-	 export let title = '';
+	export let title = '';
 	export let pagetitle = '';
 </script>
 

src/lib/helpers/http.js

@@ -184,6 +184,8 @@ function skipLoader(config) {
         new RegExp('http(s*)://(.*?)/knowledge/(.*?)/search', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/create', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/page', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/upload', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/tokenize', 'g'),
         new RegExp('http(s*)://(.*?)/users', 'g'),
         new RegExp('http(s*)://(.*?)/instruct/(.*?)', 'g'),
         new RegExp('http(s*)://(.*?)/agent/(.*?)/code-scripts', 'g'),
@@ -227,6 +229,9 @@ function skipLoader(config) {
         new RegExp('http(s*)://(.*?)/llm-provider/(.*?)/models', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/vector/collections', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/exist', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/document/processors', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/tokenizer/providers', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/tokenizer/data-providers', 'g'),
         new RegExp('http(s*)://(.*?)/logger/instruction/log', 'g'),
         new RegExp('http(s*)://(.*?)/logger/instruction/log/keys', 'g'),
         new RegExp('http(s*)://(.*?)/logger/conversation/(.*?)/content-log', 'g'),

src/lib/helpers/types/commonTypes.js

@@ -82,4 +82,10 @@
  * @property {any[]} fail
  */
 
+/**
+ * @typedef {Object} TableItemConfig
+ * @property {string} dataName
+ * @property {string} displayName
+ */
+
 export default {};

src/lib/helpers/types/knowledgeTypes.js

@@ -98,6 +98,7 @@
 /**
  * @typedef {Object} VectorKnowledgeUploadRequest
  * @property {import('$fileTypes').FileModel[]} files - The files.
+ * @property {any} [options]
  */
 
 /**
@@ -160,4 +161,33 @@
  * @property {string} field_schema_type
  */
 
+/**
+ * @typedef {Object} TokenizeRequest
+ * @property {string} text
+ * @property {string?} [provider]
+ * @property {TokenizeOptions?} [options]
+ */
+
+/**
+ * @typedef {Object} TokenizeOptions
+ * @property {string[]?} [data_providers]
+ * @property {number?} [max_ngram]
+ * @property {number?} [cutoff]
+ * @property {number?} [top_k]
+ */
+
+/**
+ * @typedef {Object} TokenizeResponse
+ * @property {TokenizeResult[]} [results]
+ * @property {boolean?} [success]
+ * @property {string?} [error_message]
+ */
+
+/**
+ * @typedef {Object} TokenizeResult
+ * @property {string} token
+ * @property {string?} [canonical_text]
+ * @property {any} data
+ */
+
 export default {};

src/lib/helpers/utils/common.js

@@ -161,11 +161,12 @@ export function scrollToBottom(container, behavior = 'smooth') {
 
 /**
  * @param {string} str
+ * @param {string} separator
  */
-export function splitTextByCase(str) {
+export function splitTextByCase(str, separator = '_') {
     if (!str) return str;
 
-    let words = str.split("_");
+    let words = str.split(separator);
     if (words.length === 1) {
         // split by camel case
         words = str.split(/(?=[A-Z])/);

src/lib/scss/custom/pages/_knowledgebase.scss

@@ -288,10 +288,11 @@
         display: flex;
         gap: 5px;
         justify-content: flex-end;
-        min-width: 300px;
+        flex-wrap: wrap;
 
         .collection-dropdown {
-            width: 100%;
+            flex: 1 1 auto;
+            min-width: 150px;
         }
 
         .collection-action-btn {
@@ -304,6 +305,17 @@
             outline: none !important;
             box-shadow: none !important;
         }
+
+        @media (max-width: 350px) {
+            flex-direction: column;
+            justify-content: center;
+            gap: 5px;
+
+            .collection-dropdown {
+                width: 100%;
+                min-width: 100%;
+            }
+        }
     }
 
     .collection-add-container {
@@ -351,6 +363,9 @@
         border-color: var(--#{$prefix}light) !important;
         background-color: var(--#{$prefix}light) !important;
         position: relative;
+        overflow-y: auto;
+        scrollbar-width: none;
+        max-height: 300px;
 
         ul {
             margin: 15px 0px;
@@ -385,15 +400,16 @@
 
         .more-detail-list {
             margin: 1px 0px 15px 0px;
-            font-size: 10px;
+            font-size: 12px;
             padding-left: 4rem;
-            overflow-y: auto;
-            scrollbar-width: none;
-            max-height: 200px;
 
-            .more-dtail-item {
+            .more-detail-item {
                 list-style-type: square;
             }
+
+            ul {
+                font-size: 15px;
+            }
         }
     }
 }
@@ -466,7 +482,7 @@
     }
 
     .doc-upload-footer {
-        margin-top: 30px;
+        margin-top: 0px;
 
         .load-doc-btn {
             font-size: 12px;
@@ -804,4 +820,25 @@
             gap: 0;
         }
     }
+}
+
+
+.knowledge-upload-modal-container {
+    .doc-upload-body {
+        width: 100%;
+        height: 100%;
+
+        .gallery-item, .doc-drop-zone {
+            width: 100%;
+            min-height: 200px;
+        }
+
+        i {
+            font-size: 5em;
+        }
+
+        .doc-drop-icon {
+            text-align: center;
+        }
+    }
 }

src/lib/services/api-endpoints.js

@@ -113,6 +113,11 @@ export const endpoints = {
     knowledgeDocumentDeleteUrl: `${host}/knowledge/document/{collection}/delete/{fileId}`,
     knowledgeDocumentDeleteAllUrl: `${host}/knowledge/document/{collection}/delete`,
     knowledgeDocumentPageListUrl: `${host}/knowledge/document/{collection}/page`,
+    knowledgeDocumentProcessorsUrl: `${host}/knowledge/document/processors`,
+
+    tokenizersUrl: `${host}/knowledge/tokenizer/providers`,
+    tokenizerDataLoadersUrl: `${host}/knowledge/tokenizer/data-providers`,
+    tokenizeUrl: `${host}/knowledge/tokenize`,
 
     // chathub 
     chatHubUrl: `${host}/chatHub`,

src/lib/services/knowledge-base-service.js

@@ -193,6 +193,15 @@ export async function getKnowledgeDocumentPageList(collection, request) {
     return response.data;
 }
 
+/**
+ * @returns {Promise<string[]>}
+ */
+export async function getKnowledgeDocumentProcessors() {
+    const url = endpoints.knowledgeDocumentProcessorsUrl;
+    const response = await axios.get(url);
+    return response.data;
+}
+
 
 /**
  * @param {import('$knowledgeTypes').CreateVectorCollectionRequest} request
@@ -274,4 +283,37 @@ export async function deleteVectorIndexes(collection, options) {
         }
     });
     return response.data;
+}
+
+
+
+
+/**
+ * @returns {Promise<string[]>}
+ */
+export async function getTokenizers() {
+    const url = endpoints.tokenizersUrl;
+    const response = await axios.get(url);
+    return response.data;
+}
+
+/**
+ * @returns {Promise<string[]>}
+ */
+export async function getTokenizerDataLoaders() {
+    const url = endpoints.tokenizerDataLoadersUrl;
+    const response = await axios.get(url);
+    return response.data;
+}
+
+/**
+ * @param {import('$knowledgeTypes').TokenizeRequest} request
+ * @returns {Promise<import('$knowledgeTypes').TokenizeResponse>}
+ */
+export async function tokenize(request) {
+    const url = endpoints.tokenizeUrl;
+    const response = await axios.post(url, {
+        ...request
+    });
+    return response.data;
 }

src/routes/page/knowledge-base/common/table/table-item.svelte

@@ -0,0 +1,119 @@
+<script>
+    import { Button } from "@sveltestrap/sveltestrap";
+    import { fly } from 'svelte/transition';
+	import Loader from "$lib/common/Loader.svelte";
+    import JSONTree from 'svelte-json-tree';
+	import { formatObject, splitTextByCase } from "$lib/helpers/utils/common";
+
+    /** @type {any} */
+    export let item;
+
+    /** @type {import('$commonTypes').TableItemConfig[]} */
+    export let columns;
+
+    /** @type {string | null} */
+    export let detailKey = null;
+
+    /** @type {boolean} */
+    export let useJsonDisplay = false;
+
+    /** @type {boolean} */
+    export let open = false;
+
+    let isLoading = false;
+    let loadMore = false;
+
+    $: {
+        if (!open) {
+            loadMore = false;
+        }
+    }
+
+    function toggleDetail() {
+        open = !open;
+    }
+</script>
+
+{#if isLoading}
+    <Loader />
+{/if}
+
+<tr in:fly={{ y: -5, duration: 800 }}>
+    {#if columns?.length > 0}
+    {#each columns as column, idx (idx)}
+        <td style={`max-width: 50px; width: ${80 / columns.length}%;`}>
+            <div class="ellipsis">{item[column.dataName] || ''}</div>
+        </td>
+    {/each}
+    {/if}
+    <td class="knowledge-op">
+        <ul class="list-unstyled hstack gap-1 mb-0 knowledge-op-list">
+            {#if detailKey}
+            <li data-bs-toggle="tooltip" data-bs-placement="top" title="View Detail">
+                <Button
+                    class="btn btn-sm btn-soft-primary"
+                    on:click={() => toggleDetail()}
+                >
+                    {#if open}
+                        <i class="bx bx-hide" />
+                    {:else}
+                        <i class="mdi mdi-eye-outline" />
+                    {/if}
+                </Button>
+            </li>
+            {/if}
+        </ul>
+    </td>
+</tr>
+
+{#if open}
+    <tr in:fly={{ y: -5, duration: 800 }} out:fly={{ y: -5, duration: 300 }}>
+        <td colspan="12">
+            <div class="knowledge-detail">
+                <ul>
+                    {#each columns as column, idx (idx)}
+                    <li>
+                        <div class="wrappable fw-bold text-primary">
+                            {column.displayName || column.dataName}
+                        </div>
+                        <div class="wrappable">
+                            {item[column.dataName] || ''}
+                        </div>
+                    </li>
+                    {/each}
+                </ul>
+                <div class="more-detail">
+                    <Button class='toggle-btn btn-sm' color="link" on:click={() => loadMore = !loadMore}>
+                        {`${loadMore ? 'Less -' : 'More +'}`}
+                    </Button>
+                </div>
+                {#if loadMore}
+                    <ul
+                        class="more-detail-list text-secondary"
+                        in:fly={{ y: -5, duration: 300 }}
+                        out:fly={{ y: -5, duration: 200 }}
+                    >
+                        {#if detailKey && item[detailKey]}
+                            {#if useJsonDisplay}
+                                <JSONTree
+                                    value={formatObject(item[detailKey])}
+                                    defaultExpandedLevel={1}
+                                    --json-tree-number-color="var(--bs-info)"
+                                    --json-tree-boolean-color="var(--bs-info)"
+                                    --json-tree-string-color="var(--bs-info)"
+                                />
+                            {:else}
+                                {#each Object.keys(item[detailKey]) as key, idx (idx)}
+                                    <li class="more-detail-item wrappable">
+                                        <span>{splitTextByCase(key)}: </span>
+                                        <span>{item[detailKey][key]}</span>
+                                    </li>
+                                {/each}
+                            {/if}
+                        {/if}
+                    </ul>
+                {/if}
+            </div>
+        </td>
+    </tr>
+{/if}