Pr@main@perf embedded multiple domain

backend/apps/chat/api/chat.py

@@ -10,7 +10,7 @@
 
 from apps.chat.curd.chat import list_chats, get_chat_with_records, create_chat, rename_chat, \
     delete_chat, get_chat_chart_data, get_chat_predict_data, get_chat_with_records_with_data, get_chat_record_by_id, \
-    format_json_data, format_json_list_data, get_chart_config
+    format_json_data, format_json_list_data, get_chart_config, list_recent_questions
 from apps.chat.models.chat_model import CreateChat, ChatRecord, RenameChat, ChatQuestion, AxisObj
 from apps.chat.task.llm import LLMService
 from common.core.deps import CurrentAssistant, SessionDep, CurrentUser, Trans
@@ -132,6 +132,10 @@ def _err(_e: Exception):
 
     return StreamingResponse(llm_service.await_result(), media_type="text/event-stream")
 
+@router.get("/recent_questions/{datasource_id}")
+async def recommend_questions(session: SessionDep, current_user: CurrentUser, datasource_id: int):
+    return list_recent_questions(session=session, current_user=current_user, datasource_id=datasource_id)
+
 
 @router.post("/question")
 async def stream_sql(session: SessionDep, current_user: CurrentUser, request_question: ChatQuestion,

backend/apps/chat/curd/chat.py

@@ -1,5 +1,6 @@
 import datetime
 from typing import List
+from sqlalchemy import desc, func
 
 import orjson
 import sqlparse
@@ -8,7 +9,8 @@
 
 from apps.chat.models.chat_model import Chat, ChatRecord, CreateChat, ChatInfo, RenameChat, ChatQuestion, ChatLog, \
     TypeEnum, OperationEnum, ChatRecordResult
-from apps.datasource.models.datasource import CoreDatasource
+from apps.datasource.crud.recommended_problem import get_datasource_recommended, get_datasource_recommended_chart
+from apps.datasource.models.datasource import CoreDatasource, DsRecommendedProblem
 from apps.system.crud.assistant import AssistantOutDsFactory
 from common.core.deps import CurrentAssistant, SessionDep, CurrentUser
 from common.utils.utils import extract_nested_json
@@ -34,6 +36,21 @@ def list_chats(session: SessionDep, current_user: CurrentUser) -> List[Chat]:
     return chart_list
 
 
+def list_recent_questions(session: SessionDep, current_user: CurrentUser, datasource_id: int) -> List[str]:
+    chat_records = (
+        session.query(ChatRecord.question)
+        .filter(
+            ChatRecord.datasource == datasource_id,
+            ChatRecord.question.isnot(None)
+        )
+        .group_by(ChatRecord.question)
+        .order_by(desc(func.max(ChatRecord.create_time)))
+        .limit(10)
+        .all()
+    )
+    return [record[0] for record in chat_records] if chat_records else []
+
+
 def rename_chat(session: SessionDep, rename_object: RenameChat) -> str:
     chat = session.get(Chat, rename_object.id)
     if not chat:
@@ -70,6 +87,7 @@ def get_chart_config(session: SessionDep, chart_record_id: int):
             pass
     return {}
 
+
 def format_chart_fields(chart_info: dict):
     fields = []
     if chart_info.get('columns') and len(chart_info.get('columns')) > 0:
@@ -88,6 +106,7 @@ def format_chart_fields(chart_info: dict):
                 fields.append(column_str)
     return fields
 
+
 def get_last_execute_sql_error(session: SessionDep, chart_id: int):
     stmt = select(ChatRecord.error).where(and_(ChatRecord.chat_id == chart_id)).order_by(
         ChatRecord.create_time.desc()).limit(1)
@@ -396,6 +415,12 @@ def create_chat(session: SessionDep, current_user: CurrentUser, create_chat_obj:
         record.finish = True
         record.create_time = datetime.datetime.now()
         record.create_by = current_user.id
+        if ds.recommended_config == 2:
+            questions = get_datasource_recommended_chart(session, ds.id)
+            record.recommended_question = orjson.dumps(questions).decode()
+            record.recommended_question_answer = orjson.dumps({
+                "content": questions
+            }).decode()
 
         _record = ChatRecord(**record.model_dump())
 

backend/apps/chat/models/chat_model.py

@@ -185,8 +185,12 @@ class AiModelQuestion(BaseModel):
 
     def sql_sys_question(self, db_type: Union[str, DB], enable_query_limit: bool = True):
         _sql_template = get_sql_example_template(db_type)
-        _query_limit = get_sql_template()['query_limit'] if enable_query_limit else get_sql_template()['no_query_limit']
-        _base_sql_rules = _sql_template['quot_rule'] + _query_limit + _sql_template['limit_rule'] + _sql_template['other_rule']
+        _base_template = get_sql_template()
+        _process_check = _sql_template.get('process_check') if _sql_template.get('process_check') else _base_template[
+            'process_check']
+        _query_limit = _base_template['query_limit'] if enable_query_limit else _base_template['no_query_limit']
+        _base_sql_rules = _sql_template['quot_rule'] + _query_limit + _sql_template['limit_rule'] + _sql_template[
+            'other_rule']
         _sql_examples = _sql_template['basic_example']
         _example_engine = _sql_template['example_engine']
         _example_answer_1 = _sql_template['example_answer_1_with_limit'] if enable_query_limit else _sql_template[
@@ -195,19 +199,20 @@ def sql_sys_question(self, db_type: Union[str, DB], enable_query_limit: bool = T
             'example_answer_2']
         _example_answer_3 = _sql_template['example_answer_3_with_limit'] if enable_query_limit else _sql_template[
             'example_answer_3']
-        return get_sql_template()['system'].format(engine=self.engine, schema=self.db_schema, question=self.question,
-                                                   lang=self.lang, terminologies=self.terminologies,
-                                                   data_training=self.data_training, custom_prompt=self.custom_prompt,
-                                                   base_sql_rules=_base_sql_rules,
-                                                   basic_sql_examples=_sql_examples,
-                                                   example_engine=_example_engine,
-                                                   example_answer_1=_example_answer_1,
-                                                   example_answer_2=_example_answer_2,
-                                                   example_answer_3=_example_answer_3)
-
-    def sql_user_question(self, current_time: str):
+        return _base_template['system'].format(engine=self.engine, schema=self.db_schema, question=self.question,
+                                               lang=self.lang, terminologies=self.terminologies,
+                                               data_training=self.data_training, custom_prompt=self.custom_prompt,
+                                               process_check=_process_check,
+                                               base_sql_rules=_base_sql_rules,
+                                               basic_sql_examples=_sql_examples,
+                                               example_engine=_example_engine,
+                                               example_answer_1=_example_answer_1,
+                                               example_answer_2=_example_answer_2,
+                                               example_answer_3=_example_answer_3)
+
+    def sql_user_question(self, current_time: str, change_title: bool):
         return get_sql_template()['user'].format(engine=self.engine, schema=self.db_schema, question=self.question,
-                                                 rule=self.rule, current_time=current_time, error_msg=self.error_msg)
+                                                 rule=self.rule, current_time=current_time, error_msg=self.error_msg,change_title = change_title)
 
     def chart_sys_question(self):
         return get_chart_template()['system'].format(sql=self.sql, question=self.question, lang=self.lang)

backend/apps/chat/task/llm.py

@@ -524,7 +524,7 @@ def select_datasource(self, _session: Session):
     def generate_sql(self, _session: Session):
         # append current question
         self.sql_message.append(HumanMessage(
-            self.chat_question.sql_user_question(current_time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))
+            self.chat_question.sql_user_question(current_time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),change_title = self.change_title)))
 
         self.current_logs[OperationEnum.GENERATE_SQL] = start_log(session=_session,
                                                                   ai_modal_id=self.chat_question.ai_modal_id,
@@ -756,6 +756,26 @@ def get_chart_type_from_sql_answer(res: str) -> Optional[str]:
 
         return chart_type
 
+    @staticmethod
+    def get_brief_from_sql_answer(res: str) -> Optional[str]:
+        json_str = extract_nested_json(res)
+        if json_str is None:
+            return None
+
+        brief: Optional[str]
+        data: dict
+        try:
+            data = orjson.loads(json_str)
+
+            if data['success']:
+                brief = data['brief']
+            else:
+                return None
+        except Exception:
+            return None
+
+        return brief
+
     def check_save_sql(self, session: Session, res: str) -> str:
         sql, *_ = self.check_sql(res=res)
         save_sql(session=session, sql=sql, record_id=self.record.id)
@@ -925,17 +945,6 @@ def run_task(self, in_chat: bool = True, stream: bool = True,
             if not stream:
                 json_result['record_id'] = self.get_record().id
 
-            # return title
-            if self.change_title:
-                if self.chat_question.question and self.chat_question.question.strip() != '':
-                    brief = rename_chat(session=_session,
-                                        rename_object=RenameChat(id=self.get_record().chat_id,
-                                                                 brief=self.chat_question.question.strip()[:20]))
-                    if in_chat:
-                        yield 'data:' + orjson.dumps({'type': 'brief', 'brief': brief}).decode() + '\n\n'
-                    if not stream:
-                        json_result['title'] = brief
-
                 # select datasource if datasource is none
             if not self.ds:
                 ds_res = self.select_datasource(_session)
@@ -981,6 +990,19 @@ def run_task(self, in_chat: bool = True, stream: bool = True,
 
             chart_type = self.get_chart_type_from_sql_answer(full_sql_text)
 
+            # return title
+            if self.change_title:
+                llm_brief = self.get_brief_from_sql_answer(full_sql_text)
+                if (llm_brief and llm_brief != '')  or (self.chat_question.question and self.chat_question.question.strip() != ''):
+                    save_brief = llm_brief if (llm_brief and llm_brief != '') else self.chat_question.question.strip()[:20]
+                    brief = rename_chat(session=_session,
+                                        rename_object=RenameChat(id=self.get_record().chat_id,
+                                                                 brief=save_brief))
+                    if in_chat:
+                        yield 'data:' + orjson.dumps({'type': 'brief', 'brief': brief}).decode() + '\n\n'
+                    if not stream:
+                        json_result['title'] = brief
+
             use_dynamic_ds: bool = self.current_assistant and self.current_assistant.type in dynamic_ds_types
             is_page_embedded: bool = self.current_assistant and self.current_assistant.type == 4
             dynamic_sql_result = None
@@ -1047,7 +1069,7 @@ def run_task(self, in_chat: bool = True, stream: bool = True,
             if in_chat:
                 yield 'data:' + orjson.dumps({'content': 'execute-success', 'type': 'sql-data'}).decode() + '\n\n'
             if not stream:
-                json_result['data'] = result.get('data')
+                json_result['data'] = get_chat_chart_data(_session, self.record.id)
 
             if finish_step.value <= ChatFinishStep.QUERY_DATA.value:
                 if stream:

backend/apps/data_training/api/data_training.py

@@ -17,6 +17,7 @@
 from common.core.config import settings
 from common.core.deps import SessionDep, CurrentUser, Trans
 from common.utils.data_format import DataFormat
+from common.utils.excel import get_excel_column_count
 
 router = APIRouter(tags=["DataTraining"], prefix="/system/data-training")
 
@@ -73,8 +74,8 @@ def inner():
             data_list.append(_data)
 
         fields = []
-        fields.append(AxisObj(name=trans('i18n_data_training.data_training'), value='question'))
-        fields.append(AxisObj(name=trans('i18n_data_training.problem_description'), value='description'))
+        fields.append(AxisObj(name=trans('i18n_data_training.problem_description'), value='question'))
+        fields.append(AxisObj(name=trans('i18n_data_training.sample_sql'), value='description'))
         fields.append(AxisObj(name=trans('i18n_data_training.effective_data_sources'), value='datasource_name'))
         if current_user.oid == 1:
             fields.append(
@@ -97,6 +98,43 @@ def inner():
     return StreamingResponse(result, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
 
 
+@router.get("/template")
+async def excel_template(trans: Trans, current_user: CurrentUser):
+    def inner():
+        data_list = []
+        _data1 = {
+            "question": '查询TEST表内所有ID',
+            "description": 'SELECT id FROM TEST',
+            "datasource_name": '生效数据源1',
+            "advanced_application_name": '生效高级应用名称',
+        }
+        data_list.append(_data1)
+
+        fields = []
+        fields.append(AxisObj(name=trans('i18n_data_training.problem_description_template'), value='question'))
+        fields.append(AxisObj(name=trans('i18n_data_training.sample_sql_template'), value='description'))
+        fields.append(AxisObj(name=trans('i18n_data_training.effective_data_sources_template'), value='datasource_name'))
+        if current_user.oid == 1:
+            fields.append(
+                AxisObj(name=trans('i18n_data_training.advanced_application_template'), value='advanced_application_name'))
+
+        md_data, _fields_list = DataFormat.convert_object_array_for_pandas(fields, data_list)
+
+        df = pd.DataFrame(md_data, columns=_fields_list)
+
+        buffer = io.BytesIO()
+
+        with pd.ExcelWriter(buffer, engine='xlsxwriter',
+                            engine_kwargs={'options': {'strings_to_numbers': False}}) as writer:
+            df.to_excel(writer, sheet_name='Sheet1', index=False)
+
+        buffer.seek(0)
+        return io.BytesIO(buffer.getvalue())
+
+    result = await asyncio.to_thread(inner)
+    return StreamingResponse(result, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
+
+
 path = settings.EXCEL_PATH
 
 from sqlalchemy.orm import sessionmaker, scoped_session
@@ -136,6 +174,9 @@ def inner():
 
         for sheet_name in sheet_names:
 
+            if get_excel_column_count(save_path, sheet_name) < len(use_cols):
+                raise Exception(trans("i18n_excel_import.col_num_not_match"))
+
             df = pd.read_excel(
                 save_path,
                 sheet_name=sheet_name,

backend/apps/datasource/api/recommended_problem.py

@@ -1,7 +1,10 @@
 from fastapi import APIRouter
 
-from apps.datasource.crud.recommended_problem import get_datasource_recommended
-from common.core.deps import SessionDep
+from apps.datasource.crud.datasource import update_ds_recommended_config
+from apps.datasource.crud.recommended_problem import get_datasource_recommended, \
+    save_recommended_problem
+from apps.datasource.models.datasource import RecommendedProblemBase
+from common.core.deps import SessionDep, CurrentUser
 
 router = APIRouter(tags=["recommended_problem"], prefix="/recommended_problem")
 
@@ -10,3 +13,8 @@
 async def datasource_recommended(session: SessionDep, ds_id: int):
     return get_datasource_recommended(session, ds_id)
 
+
+@router.post("/save_recommended_problem")
+async def datasource_recommended(session: SessionDep, user: CurrentUser, data_info: RecommendedProblemBase):
+    update_ds_recommended_config(session, data_info.datasource_id, data_info.recommended_config)
+    return save_recommended_problem(session, user, data_info)

backend/apps/datasource/crud/datasource.py

@@ -109,6 +109,11 @@ def update_ds(session: SessionDep, trans: Trans, user: CurrentUser, ds: CoreData
     run_save_ds_embeddings([ds.id])
     return ds
 
+def update_ds_recommended_config(session: SessionDep,datasource_id: int, recommended_config:int):
+    record = session.exec(select(CoreDatasource).where(CoreDatasource.id == datasource_id)).first()
+    record.recommended_config = recommended_config
+    session.add(record)
+    session.commit()
 
 def delete_ds(session: SessionDep, id: int):
     term = session.exec(select(CoreDatasource).where(CoreDatasource.id == id)).first()

backend/apps/datasource/crud/recommended_problem.py

@@ -1,11 +1,31 @@
+import datetime
+
 from sqlmodel import select
 
-from common.core.deps import SessionDep
-from ..models.datasource import DsRecommendedProblem
+from common.core.deps import SessionDep, CurrentUser, Trans
+from ..models.datasource import DsRecommendedProblem, RecommendedProblemBase, RecommendedProblemBaseChat
 
 
 def get_datasource_recommended(session: SessionDep, ds_id: int):
     statement = select(DsRecommendedProblem).where(DsRecommendedProblem.datasource_id == ds_id)
-    dsRecommendedProblem = session.exec(statement)
+    dsRecommendedProblem = session.exec(statement).all()
     return dsRecommendedProblem
 
+def get_datasource_recommended_chart(session: SessionDep, ds_id: int):
+    statement = select(DsRecommendedProblem.question).where(DsRecommendedProblem.datasource_id == ds_id)
+    dsRecommendedProblems = session.exec(statement).all()
+    return dsRecommendedProblems
+
+def save_recommended_problem(session: SessionDep,user: CurrentUser, data_info: RecommendedProblemBase):
+    session.query(DsRecommendedProblem).filter(DsRecommendedProblem.datasource_id == data_info.datasource_id).delete(synchronize_session=False)
+    problemInfo = data_info.problemInfo
+    if problemInfo is not None:
+        for problemItem in problemInfo:
+            problemItem.id = None
+            problemItem.create_time = datetime.datetime.now()
+            problemItem.create_by = user.id
+            record = DsRecommendedProblem(**problemItem.model_dump())
+            session.add(record)
+            session.flush()
+            session.refresh(record)
+    session.commit()

backend/apps/datasource/models/datasource.py

@@ -74,6 +74,18 @@ class CreateDatasource(BaseModel):
     tables: List[CoreTable] = []
     recommended_config: int = 1
 
+class RecommendedProblemBase(BaseModel):
+    datasource_id: int = None
+    recommended_config: int = None
+    problemInfo: List[DsRecommendedProblem] = []
+
+
+class RecommendedProblemBaseChat:
+    def __init__(self, content):
+        self.content = content
+
+    content: List[str] = []
+
 
 # edit local saved table and fields
 class TableObj(BaseModel):