Skip to content

feat: Terminology / SQL Sample Management add enabled control #407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ulleo merged 1 commit into from
Nov 6, 2025
Merged

feat: Terminology / SQL Sample Management add enabled control #407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions backend/apps/data_training/curd/data_training.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ def page_data_training(session: SessionDep, current_page: int = 1, page_size: in
DataTraining.question,
DataTraining.create_time,
DataTraining.description,
DataTraining.enabled,
)
.outerjoin(CoreDatasource, and_(DataTraining.datasource == CoreDatasource.id))
.where(and_(DataTraining.id.in_(paginated_parent_ids)))
Expand All @@ -79,6 +80,7 @@ def page_data_training(session: SessionDep, current_page: int = 1, page_size: in
question=row.question,
create_time=row.create_time,
description=row.description,
enabled=row.enabled,
))

return current_page, page_size, total_count, total_pages, _list
Expand Down Expand Up @@ -221,11 +223,11 @@ def save_embeddings(session_maker, ids: List[int]):
embedding_sql = f"""
SELECT id, datasource, question, similarity
FROM
(SELECT id, datasource, question, oid,
(SELECT id, datasource, question, oid, enabled,
( 1 - (embedding <=> :embedding_array) ) AS similarity
FROM data_training AS child
) TEMP
WHERE similarity > {settings.EMBEDDING_DATA_TRAINING_SIMILARITY} and oid = :oid and datasource = :datasource
WHERE similarity > {settings.EMBEDDING_DATA_TRAINING_SIMILARITY} and oid = :oid and datasource = :datasource and enabled = true
ORDER BY similarity DESC
LIMIT {settings.EMBEDDING_DATA_TRAINING_TOP_COUNT}
"""
Expand All @@ -246,7 +248,8 @@ def select_training_by_question(session: SessionDep, question: str, oid: int, da
.where(
and_(or_(text(":sentence ILIKE '%' || question || '%'"), text("question ILIKE '%' || :sentence || '%'")),
DataTraining.oid == oid,
DataTraining.datasource == datasource)
DataTraining.datasource == datasource,
DataTraining.enabled == True,)
)
)

Expand Down
23 changes: 14 additions & 9 deletions backend/apps/terminology/curd/terminology.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
Terminology.specific_ds,
Terminology.datasource_ids,
children_subquery.c.other_words,
func.jsonb_agg(CoreDatasource.name).filter(CoreDatasource.id.isnot(None)).label('datasource_names')
func.jsonb_agg(CoreDatasource.name).filter(CoreDatasource.id.isnot(None)).label('datasource_names'),
Terminology.enabled
)
.outerjoin(
children_subquery,
Expand All @@ -122,7 +123,8 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
Terminology.description,
Terminology.specific_ds,
Terminology.datasource_ids,
children_subquery.c.other_words
children_subquery.c.other_words,
Terminology.enabled
)
.order_by(Terminology.create_time.desc())
)
Expand Down Expand Up @@ -175,7 +177,8 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
Terminology.specific_ds,
Terminology.datasource_ids,
children_subquery.c.other_words,
func.jsonb_agg(CoreDatasource.name).filter(CoreDatasource.id.isnot(None)).label('datasource_names')
func.jsonb_agg(CoreDatasource.name).filter(CoreDatasource.id.isnot(None)).label('datasource_names'),
Terminology.enabled
)
.outerjoin(
children_subquery,
Expand All @@ -197,7 +200,8 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
Terminology.description,
Terminology.specific_ds,
Terminology.datasource_ids,
children_subquery.c.other_words
children_subquery.c.other_words,
Terminology.enabled
)
.order_by(Terminology.create_time.desc())
)
Expand All @@ -214,6 +218,7 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
specific_ds=row.specific_ds if row.specific_ds is not None else False,
datasource_ids=row.datasource_ids if row.datasource_ids is not None else [],
datasource_names=row.datasource_names if row.datasource_names is not None else [],
enabled=row.enabled if row.enabled is not None else False,
))

return current_page, page_size, total_count, total_pages, _list
Expand Down Expand Up @@ -474,11 +479,11 @@ def save_embeddings(session_maker, ids: List[int]):
embedding_sql = f"""
SELECT id, pid, word, similarity
FROM
(SELECT id, pid, word, oid, specific_ds, datasource_ids,
(SELECT id, pid, word, oid, specific_ds, datasource_ids, enabled,
( 1 - (embedding <=> :embedding_array) ) AS similarity
FROM terminology AS child
) TEMP
WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid
WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid AND enabled = true
AND (specific_ds = false OR specific_ds IS NULL)
ORDER BY similarity DESC
LIMIT {settings.EMBEDDING_TERMINOLOGY_TOP_COUNT}
Expand All @@ -487,11 +492,11 @@ def save_embeddings(session_maker, ids: List[int]):
embedding_sql_with_datasource = f"""
SELECT id, pid, word, similarity
FROM
(SELECT id, pid, word, oid, specific_ds, datasource_ids,
(SELECT id, pid, word, oid, specific_ds, datasource_ids, enabled,
( 1 - (embedding <=> :embedding_array) ) AS similarity
FROM terminology AS child
) TEMP
WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid
WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid AND enabled = true
AND (
(specific_ds = false OR specific_ds IS NULL)
OR
Expand All @@ -515,7 +520,7 @@ def select_terminology_by_word(session: SessionDep, word: str, oid: int, datasou
Terminology.word,
)
.where(
and_(text(":sentence ILIKE '%' || word || '%'"), Terminology.oid == oid)
and_(text(":sentence ILIKE '%' || word || '%'"), Terminology.oid == oid, Terminology.enabled == True)
)
)

Expand Down