diff --git a/AGENTS.md b/AGENTS.md index 6ac2717f..c0f643f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,3 +62,20 @@ SQLBot 让业务用户用自然语言提问，基于已配置的数据源生成 - 按验收条件选择验证层级，不把构建或单元测试等同于产品验收。普通任务不默认构建镜像或启动完整运行栈；已要求真实接口、浏览器或部署验证时，可启动必要的隔离服务。环境范围、外部成本或数据用途不明确时先确认。 - 除非用户明确要求，不要上传、发布或推送镜像 / wheel / 包。 - 变更涉及本文件或 `docs/agents/` 描述的约定（目录职责、命令、流程）时，同步更新对应文档。 + +## Agent skills + +### Issue tracker + +处理 Issue 或发布规格、任务时，使用 dataease/SQLBot 的 GitHub Issues； +操作约定见 `docs/agents/issue-tracker.md`。 + +### Triage labels + +执行 triage 时，使用五个默认角色标签； +映射见 `docs/agents/triage-labels.md`。 + +### Domain docs + +工程技能采用 single-context 布局；探索相关领域前， +按 `docs/agents/domain.md` 读取词汇表和架构决策。 diff --git a/CONTEXT.md b/CONTEXT.md index b47b597a..3e4367e2 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -54,6 +54,18 @@ _Avoid_: Terminology、SQL example 术语、SQL 示例和自定义提示词生效的边界：工作空间、数据源或高级应用。高级应用是独立知识域，不继承工作空间级知识。 _Avoid_: 权限——作用域决定向模型注入哪些知识，权限决定用户能访问哪些数据 +**Knowledge Retrieval / 知识召回**: +根据问题匹配相关术语和 SQL 示例，为问数提供业务解释与查询示例。字面匹配与向量匹配是补充关系。 +_Avoid_: 自定义提示词注入、全系统向量化 + +**Table Selection / 表筛选**: +从当前可访问的候选表中选择与问题相关的表，为生成 SQL 提供表结构上下文。 +_Avoid_: 表权限——筛选不授予访问权限 + +**Datasource Preselection / 数据源预筛选**: +在自动选择数据源之前，缩小与问题相关的数据源候选范围。 +_Avoid_: 数据源选择结果——预筛选结果仍是候选集合 + **Row Permission / 行权限**（xpack）: 对表追加的行级过滤条件；同一表命中多条时取 AND，经改写 SQL 执行。 _Avoid_: Column Permission diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index 4be8ed1e..2ff94400 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -52,6 +52,7 @@ from apps.terminology.curd.terminology import expand_with_terminology from apps.terminology.curd.terminology import get_terminology_template from common.core.config import settings +from common.core.embedding_config import load_embedding_config from common.core.db import engine from common.core.deps import CurrentAssistant, CurrentUser from common.error import SingleMessageError, SQLBotDBError, ParseSQLResultError, SQLBotDBConnectionError @@ -125,6 +126,7 @@ class LLMService: def __init__(self, session: Session, current_user: CurrentUser, chat_question: ChatQuestion, current_assistant: Optional[CurrentAssistant] = None, no_reasoning: bool = False, embedding: bool = False, config: LLMConfig = None): + self.embedding_config = load_embedding_config(session) self.sql_message = [] self.chart_message = [] self.generate_sql_logs = [] @@ -155,6 +157,9 @@ def __init__(self, session: Session, current_user: CurrentUser, chat_question: C raise SingleMessageError("Current assistant user cannot not access this chat") self.current_user.oid = chat.oid + SQLBotLogUtil.info( + f"[embedding] stage=config chat_id={chat_id} config={self.embedding_config}" + ) ds: CoreDatasource | AssistantOutDsSchema | None = None if not chat.datasource and chat_question.datasource_id: _ds = session.get(CoreDatasource, chat_question.datasource_id) @@ -386,13 +391,19 @@ def filter_terminology_template(self, _session: Session, oid: int = None, ds_id: self.chat_question.terminologies, term_list = get_terminology_template(_session, match_text, calculate_oid, - None, self.current_assistant.id) + None, self.current_assistant.id, config=self.embedding_config) else: self.chat_question.terminologies, term_list = get_terminology_template(_session, match_text, calculate_oid, - calculate_ds_id) + calculate_ds_id, config=self.embedding_config) + SQLBotLogUtil.info( + f"[embedding] stage=terminology chat_id={self.chat_question.chat_id} " + f"record_id={self.record.id} " + f"retrieval_mode={'literal_and_vector' if self.embedding_config.enabled else 'literal_only'} " + f"result_count={len(term_list)}" + ) self.current_logs[OperationEnum.FILTER_TERMS] = end_log(session=_session, log=self.current_logs[OperationEnum.FILTER_TERMS], full_message=term_list) @@ -440,12 +451,18 @@ def filter_training_template(self, _session: Session, oid: int = None, ds_id: in self.chat_question.data_training, example_list = get_training_template(_session, self.chat_question.question, calculate_oid, - None, self.current_assistant.id) + None, self.current_assistant.id, config=self.embedding_config) else: self.chat_question.data_training, example_list = get_training_template(_session, self.chat_question.question, calculate_oid, - calculate_ds_id) + calculate_ds_id, config=self.embedding_config) + SQLBotLogUtil.info( + f"[embedding] stage=sql_example chat_id={self.chat_question.chat_id} " + f"record_id={self.record.id} " + f"retrieval_mode={'literal_and_vector' if self.embedding_config.enabled else 'literal_only'} " + f"result_count={len(example_list)}" + ) self.current_logs[OperationEnum.FILTER_SQL_EXAMPLE] = end_log(session=_session, log=self.current_logs[ OperationEnum.FILTER_SQL_EXAMPLE], @@ -552,7 +569,15 @@ def choose_table_schema(self, _session: Session): current_user=self.current_user, ds=self.ds, question=self.chat_question.question, - keywords=keywords) + keywords=keywords, config=self.embedding_config) + + SQLBotLogUtil.info( + f"[embedding] stage=table_context chat_id={self.chat_question.chat_id} " + f"record_id={self.record.id} ds_id={self.ds.id} " + f"source={'dynamic_datasource' if self.out_ds_instance else 'configured_datasource'} " + f"vector_enabled={self.embedding_config.table_enabled and not self.out_ds_instance} " + f"result_count={len(tables)}" + ) # Get sample data for all tables if not self.out_ds_instance and settings.SAMPLE_DATA_ENABLED: @@ -767,16 +792,28 @@ def select_datasource(self, _session: Session): if not _ds_list: raise SingleMessageError('No available datasource configuration found') ignore_auto_select = _ds_list and len(_ds_list) == 1 + if ignore_auto_select: + SQLBotLogUtil.info( + f"[embedding] stage=datasource chat_id={self.chat_question.chat_id} " + f"record_id={self.record.id} action=skip reason=single_candidate candidate_count=1" + ) # ignore auto select ds full_thinking_text = '' full_text = '' if not ignore_auto_select: - if settings.TABLE_EMBEDDING_ENABLED and ( - not self.current_assistant or (self.current_assistant and self.current_assistant.type != 1)): + if self.embedding_config.table_enabled and ( + not self.current_assistant or (self.current_assistant and self.current_assistant.type not in dynamic_ds_types)): _ds_list = get_ds_embedding(_session, _ds_list, self.out_ds_instance, - self.chat_question.question, self.current_assistant) + self.chat_question.question, self.current_assistant, config=self.embedding_config) # yield {'content': '{"id":' + str(ds.get('id')) + '}'} + else: + reason = 'disabled' if not self.embedding_config.table_enabled else 'dynamic_datasource' + SQLBotLogUtil.info( + f"[embedding] stage=datasource chat_id={self.chat_question.chat_id} " + f"record_id={self.record.id} action=skip reason={reason} " + f"candidate_count={len(_ds_list)} result_count={len(_ds_list)}" + ) _ds_list_dict = [] for _ds in _ds_list: @@ -873,7 +910,7 @@ def select_datasource(self, _session: Session): except Exception as e: _error = e - if not ignore_auto_select and not settings.TABLE_EMBEDDING_ENABLED: + if not ignore_auto_select and not self.embedding_config.table_enabled: self.record = save_select_datasource_answer(session=_session, record_id=self.record.id, answer=orjson.dumps({'content': full_text}).decode(), datasource=_datasource, @@ -883,7 +920,7 @@ def select_datasource(self, _session: Session): ds_id = self.ds.id if isinstance(self.ds, CoreDatasource) else None # 提取关键词（前置步骤，结果复用于术语匹配和表匹配） - if settings.TABLE_EMBEDDING_KEYWORD_ENABLED: + if self.embedding_config.keyword_enabled: is_error, kw_result = self.extract_keywords(_session) if is_error: raise SingleMessageError(kw_result) @@ -1365,7 +1402,7 @@ def run_task(self, in_chat: bool = True, stream: bool = True, ds_id = self.ds.id if isinstance(self.ds, CoreDatasource) else None # 提取关键词（前置步骤，结果复用于术语匹配和表匹配） - if settings.TABLE_EMBEDDING_KEYWORD_ENABLED: + if self.embedding_config.keyword_enabled: is_error, kw_result = self.extract_keywords(_session) if is_error: raise SingleMessageError(kw_result) diff --git a/backend/apps/data_training/curd/data_training.py b/backend/apps/data_training/curd/data_training.py index 4db2fb86..f66c0575 100644 --- a/backend/apps/data_training/curd/data_training.py +++ b/backend/apps/data_training/curd/data_training.py @@ -3,17 +3,19 @@ from typing import List, Optional from xml.dom.minidom import parseString -from sqlalchemy import and_, select, func, delete, update, or_ +from sqlalchemy import case, and_, select, func, delete, update, or_ from sqlalchemy import text from apps.ai_model.embedding import EmbeddingModelCache +from apps.datasource.embedding.utils import parse_valid_embedding from apps.data_training.models.data_training_model import DataTrainingInfo, DataTraining, DataTrainingInfoResult from apps.datasource.models.datasource import CoreDatasource from apps.system.models.system_model import AssistantModel from apps.template.generate_chart.generator import get_base_data_training_template -from common.core.config import settings +from common.core.embedding_config import EmbeddingConfig, load_embedding_config from common.core.deps import SessionDep, Trans from common.utils.dict_to_xml import dict_to_xml +from common.utils.embedding_lifecycle import generate_missing_embeddings from common.utils.embedding_threads import run_save_data_training_embeddings @@ -266,6 +268,7 @@ def update_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans raise Exception(trans("i18n_data_training.exists_in_db")) stmt = update(DataTraining).where(and_(DataTraining.id == info.id)).values( + embedding=case((DataTraining.question.is_distinct_from(info.question.strip()), None), else_=DataTraining.embedding), question=info.question.strip(), description=info.description.strip(), datasource=info.datasource, @@ -442,84 +445,46 @@ def enable_training(session: SessionDep, id: int, enabled: bool, trans: Trans, o session.commit() -# def run_save_embeddings(ids: List[int]): -# executor.submit(save_embeddings, ids) -# -# -# def fill_empty_embeddings(): -# executor.submit(run_fill_empty_embeddings) - - def run_fill_empty_embeddings(session_maker): - try: - if not settings.EMBEDDING_ENABLED: - return - - session = session_maker() - stmt = select(DataTraining.id).where(and_(DataTraining.embedding.is_(None))) - results = session.execute(stmt).scalars().all() - - save_embeddings(session_maker, results) - except Exception: - traceback.print_exc() - finally: - session_maker.remove() + return generate_missing_embeddings(session_maker, DataTraining, DataTraining.question) def save_embeddings(session_maker, ids: List[int]): - if not settings.EMBEDDING_ENABLED: - return - - if not ids or len(ids) == 0: + if not ids: return - try: - session = session_maker() - _list = session.query(DataTraining).filter(and_(DataTraining.id.in_(ids))).all() - - _question_list = [item.question for item in _list] - - model = EmbeddingModelCache.get_model() - - results = model.embed_documents(_question_list) - - for index in range(len(results)): - item = results[index] - stmt = update(DataTraining).where(and_(DataTraining.id == _list[index].id)).values(embedding=item) - session.execute(stmt) - session.commit() - - except Exception: - traceback.print_exc() - finally: - session_maker.remove() + return generate_missing_embeddings( + session_maker, DataTraining, DataTraining.question, + DataTraining.id.in_(ids), + ) -embedding_sql = f""" +embedding_sql = """ SELECT id, datasource, question, similarity FROM (SELECT id, datasource, question, oid, enabled, ( 1 - (embedding <=> :embedding_array) ) AS similarity FROM data_training AS child ) TEMP -WHERE similarity > {settings.EMBEDDING_DATA_TRAINING_SIMILARITY} and oid = :oid and datasource = :datasource and enabled = true +WHERE similarity > :similarity and oid = :oid and datasource = :datasource and enabled = true ORDER BY similarity DESC -LIMIT {settings.EMBEDDING_DATA_TRAINING_TOP_COUNT} +LIMIT :top_count """ -embedding_sql_in_advanced_application = f""" +embedding_sql_in_advanced_application = """ SELECT id, advanced_application, question, similarity FROM (SELECT id, advanced_application, question, oid, enabled, ( 1 - (embedding <=> :embedding_array) ) AS similarity FROM data_training AS child ) TEMP -WHERE similarity > {settings.EMBEDDING_DATA_TRAINING_SIMILARITY} and oid = :oid and advanced_application = :advanced_application and enabled = true +WHERE similarity > :similarity and oid = :oid and advanced_application = :advanced_application and enabled = true ORDER BY similarity DESC -LIMIT {settings.EMBEDDING_DATA_TRAINING_TOP_COUNT} +LIMIT :top_count """ def select_training_by_question(session: SessionDep, question: str, oid: int, datasource: Optional[int] = None, - advanced_application_id: Optional[int] = None): + advanced_application_id: Optional[int] = None, *, config: EmbeddingConfig | None = None): + config = config or load_embedding_config(session) if question.strip() == "": return [] @@ -547,20 +512,20 @@ def select_training_by_question(session: SessionDep, question: str, oid: int, da for row in results: _list.append(DataTraining(id=row.id, question=row.question)) - if settings.EMBEDDING_ENABLED: + if config.enabled: with session.begin_nested(): try: model = EmbeddingModelCache.get_model() - embedding = model.embed_query(question) + embedding = parse_valid_embedding(model.embed_query(question)) if advanced_application_id is not None: results = session.execute(text(embedding_sql_in_advanced_application), - {'embedding_array': str(embedding), 'oid': oid, + {'embedding_array': str(embedding), 'oid': oid, 'similarity': config.data_training_similarity, 'top_count': config.data_training_top_count, 'advanced_application': advanced_application_id}) else: results = session.execute(text(embedding_sql), - {'embedding_array': str(embedding), 'oid': oid, 'datasource': datasource}) + {'embedding_array': str(embedding), 'oid': oid, 'similarity': config.data_training_similarity, 'top_count': config.data_training_top_count, 'datasource': datasource}) for row in results: _list.append(DataTraining(id=row.id, question=row.question)) @@ -617,12 +582,14 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'sql-examples') -> str: def get_training_template(session: SessionDep, question: str, oid: Optional[int] = 1, datasource: Optional[int] = None, - advanced_application_id: Optional[int] = None) -> tuple[str, list[dict]]: + advanced_application_id: Optional[int] = None, *, + config: EmbeddingConfig | None = None) -> tuple[str, list[dict]]: + config = config or load_embedding_config(session) if not oid: oid = 1 if not datasource and not advanced_application_id: return '', [] - _results = select_training_by_question(session, question, oid, datasource, advanced_application_id) + _results = select_training_by_question(session, question, oid, datasource, advanced_application_id, config=config) if _results and len(_results) > 0: data_training = to_xml_string(_results) template = get_base_data_training_template().format(data_training=data_training) diff --git a/backend/apps/datasource/api/datasource.py b/backend/apps/datasource/api/datasource.py index 6f91cc22..36448557 100644 --- a/backend/apps/datasource/api/datasource.py +++ b/backend/apps/datasource/api/datasource.py @@ -1,3 +1,5 @@ +from apps.datasource.crud.embedding_state import invalidate_metadata +from common.utils.embedding_threads import run_save_table_embeddings, run_save_ds_embeddings import asyncio import hashlib import io @@ -505,6 +507,14 @@ async def upload_ds_schema(session: SessionDep, id: int = Path(..., description= # sheet table mapping sheet_table_map = {} + changed_ids = [] + if table_sheet: + names = [table[t_n_col] for table in table_sheet] + changed_ids = session.query(CoreTable.id).filter( + CoreTable.ds_id == id, CoreTable.table_name.in_(names)).all() + changed_ids = [row[0] for row in changed_ids] + invalidate_metadata(session, id, changed_ids) + # get data and update # update table comment if table_sheet and len(table_sheet) > 0: @@ -530,9 +540,12 @@ async def upload_ds_schema(session: SessionDep, id: int = Path(..., description= CoreField.field_name == field[f_n_col])).update( {'custom_comment': field[f_c_col]}) session.commit() + run_save_table_embeddings(changed_ids) + run_save_ds_embeddings([id]) return True except Exception as e: + session.rollback() raise HTTPException(status_code=500, detail=f"Parse Excel Failed: {str(e)}") diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 67bb9759..1e583615 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -14,11 +14,12 @@ from apps.db.db import get_tables, get_fields, exec_sql, check_connection from apps.db.engine import get_engine_config, get_engine_conn from apps.system.schemas.auth import CacheName, CacheNamespace -from common.core.config import settings +from common.core.embedding_config import EmbeddingConfig, load_embedding_config from common.core.deps import SessionDep, CurrentUser, Trans from common.utils.embedding_threads import run_save_table_embeddings, run_save_ds_embeddings from common.utils.utils import SQLBotLogUtil, deepcopy_ignore_extra, equals_ignore_case from common.core.sqlbot_cache import cache, clear_cache +from .embedding_state import invalidate_metadata from .table import get_tables_by_ds_id from ..crud.field import delete_field_by_ds_id, update_field from ..crud.table import delete_table_by_ds_id, update_table @@ -104,7 +105,8 @@ def update_ds(session: SessionDep, trans: Trans, user: CurrentUser, ds: CoreData # status = check_status(session, trans, ds) ds.status = "Success" record = session.exec(select(CoreDatasource).where(CoreDatasource.id == ds.id)).first() - update_data = ds.model_dump(exclude_unset=True) + invalidate_metadata(session, record.id) + update_data = ds.model_dump(exclude_unset=True, exclude={"embedding"}) for field, value in update_data.items(): setattr(record, field, value) session.add(record) @@ -206,6 +208,7 @@ def sync_single_fields(session: SessionDep, trans: Trans, ds_id: int, id: int): def sync_table(session: SessionDep, ds: CoreDatasource, tables: List[CoreTable]): + invalidate_metadata(session, ds.id, [t.id for t in get_tables_by_ds_id(session, ds.id)]) id_list = [] for item in tables: statement = select(CoreTable).where(and_(CoreTable.ds_id == ds.id, CoreTable.table_name == item.table_name)) @@ -217,7 +220,7 @@ def sync_table(session: SessionDep, ds: CoreDatasource, tables: List[CoreTable]) record.table_comment = item.table_comment session.add(record) - session.commit() + session.flush() else: # save new table table = CoreTable(ds_id=ds.id, checked=True, table_name=item.table_name, table_comment=item.table_comment, @@ -227,11 +230,11 @@ def sync_table(session: SessionDep, ds: CoreDatasource, tables: List[CoreTable]) session.refresh(table) item.id = table.id id_list.append(table.id) - session.commit() + session.flush() # sync field fields = getFieldsByDs(session, ds, item.table_name) - sync_fields(session, ds, item, fields) + sync_fields(session, ds, item, fields, commit=False) if len(id_list) > 0: session.query(CoreTable).filter(and_(CoreTable.ds_id == ds.id, CoreTable.id.not_in(id_list))).delete( @@ -249,7 +252,8 @@ def sync_table(session: SessionDep, ds: CoreDatasource, tables: List[CoreTable]) run_save_ds_embeddings([ds.id]) -def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, fields: List[ColumnSchema]): +def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, fields: List[ColumnSchema], *, commit=True): + invalidate_metadata(session, ds.id, [table.id]) id_list = [] for index, item in enumerate(fields): statement = select(CoreField).where( @@ -263,7 +267,7 @@ def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, field record.field_index = index record.field_type = item.fieldType session.add(record) - session.commit() + session.flush() else: field = CoreField(ds_id=ds.id, table_id=table.id, checked=True, field_name=item.fieldName, field_type=item.fieldType, field_comment=item.fieldComment, @@ -273,11 +277,13 @@ def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, field session.refresh(field) item.id = field.id id_list.append(field.id) - session.commit() + session.flush() - if len(id_list) > 0: - session.query(CoreField).filter(and_(CoreField.table_id == table.id, CoreField.id.not_in(id_list))).delete( - synchronize_session=False) + stale_fields = session.query(CoreField).filter(CoreField.table_id == table.id) + if id_list: + stale_fields = stale_fields.filter(CoreField.id.not_in(id_list)) + stale_fields.delete(synchronize_session=False) + if commit: session.commit() @@ -415,9 +421,6 @@ def updateNum(session: SessionDep, ds: CoreDatasource): num = f'{len(selected_tables)}/{len(all_tables)}' record = session.exec(select(CoreDatasource).where(CoreDatasource.id == ds.id)).first() - update_data = ds.model_dump(exclude_unset=True) - for field, value in update_data.items(): - setattr(record, field, value) record.num = num session.add(record) session.commit() @@ -530,7 +533,7 @@ def get_tables_sample_data(session: SessionDep, current_user: CurrentUser, ds: C def get_table_schema(session: SessionDep, current_user: CurrentUser, ds: CoreDatasource, question: str, embedding: bool = True, table_list: list[str] = None, - keywords: str = None) -> tuple[str, list]: + keywords: str = None, *, config: EmbeddingConfig = None) -> tuple[str, list]: schema_str = "" table_objs = get_table_obj_by_ds(session=session, current_user=current_user, ds=ds) if len(table_objs) == 0: @@ -587,66 +590,65 @@ def get_table_schema(session: SessionDep, current_user: CurrentUser, ds: CoreDat return schema_str, [] # 执行表 embedding 匹配 - if embedding and tables and settings.TABLE_EMBEDDING_ENABLED: + if embedding and config is None: + config = load_embedding_config(session) + if embedding and tables and config.table_enabled: + candidate_count = len(tables) try: tables = calc_table_embedding(tables, question, session=session, oid=current_user.oid, - keywords=keywords) + keywords=keywords, config=config) + SQLBotLogUtil.info( + f"[embedding] stage=table ds_id={ds.id} action=selection_result vector_enabled=True " + f"candidate_count={candidate_count} result_count={len(tables)} " + f"limit={config.table_count} keyword_enabled={config.keyword_enabled} " + f"alpha={config.table_alpha}" + ) except ValueError as e: # LLM 判断意图不是查数据或关键词提取失败 return str(e), [] + elif embedding: + SQLBotLogUtil.info( + f"[embedding] stage=table ds_id={ds.id} action=skip reason=disabled " + f"candidate_count={len(tables)} result_count={len(tables)}" + ) # splice schema if tables: for s in tables: schema_str += s.get('schema_table') table_name_list.append(s.get('table_name')) - # field relation + # Complete one-hop relations only inside the original selected/authorized + # candidate and column set. Graph identifiers may be strings or integers. if tables and ds.table_relation: - relations = list(filter(lambda x: x.get('shape') == 'edge', ds.table_relation)) + eligible = {str(table['id']): table for table in all_tables} + selected = {str(table['id']) for table in tables} + visible_fields = { + (str(obj.table.id), str(field.id)): field.field_name + for obj in table_objs if str(obj.table.id) in eligible + for field in (obj.fields or []) + } + relations = [] + for relation in ds.table_relation: + if relation.get('shape') != 'edge': + continue + source, target = relation.get('source', {}), relation.get('target', {}) + source_id, target_id = str(source.get('cell')), str(target.get('cell')) + source_key = (source_id, str(source.get('port'))) + target_key = (target_id, str(target.get('port'))) + if (source_id in eligible and target_id in eligible + and (source_id in selected or target_id in selected) + and source_key in visible_fields and target_key in visible_fields): + relations.append((source_key, target_key)) + related = {key[0] for relation in relations for key in relation} + for table_id, table in eligible.items(): + if table_id in related - selected: + schema_str += table['schema_table'] + table_name_list.append(table['table_name']) if relations: - # Complete the missing table - # get tables in relation, remove irrelevant relation - embedding_table_ids = [s.get('id') for s in tables] - all_relations = list( - filter(lambda x: x.get('source').get('cell') in embedding_table_ids or x.get('target').get( - 'cell') in embedding_table_ids, relations)) - - # get relation table ids, sub embedding table ids - relation_table_ids = [] - for r in all_relations: - relation_table_ids.append(r.get('source').get('cell')) - relation_table_ids.append(r.get('target').get('cell')) - relation_table_ids = list(set(relation_table_ids)) - # get table dict - table_records = session.query(CoreTable).filter(CoreTable.id.in_(list(map(int, relation_table_ids)))).all() - table_dict = {} - for ele in table_records: - table_dict[ele.id] = ele.table_name - - # get lost table ids - lost_table_ids = list(set(relation_table_ids) - set(embedding_table_ids)) - # get lost table schema and splice it - lost_tables = list(filter(lambda x: x.get('id') in lost_table_ids, all_tables)) - if lost_tables: - for s in lost_tables: - schema_str += s.get('schema_table') - table_name_list.append(s.get('table_name')) - - # get field dict - relation_field_ids = [] - for relation in all_relations: - relation_field_ids.append(relation.get('source').get('port')) - relation_field_ids.append(relation.get('target').get('port')) - relation_field_ids = list(set(relation_field_ids)) - field_records = session.query(CoreField).filter(CoreField.id.in_(list(map(int, relation_field_ids)))).all() - field_dict = {} - for ele in field_records: - field_dict[ele.id] = ele.field_name - - if all_relations: - schema_str += '【Foreign keys】\n' - for ele in all_relations: - schema_str += f"{table_dict.get(int(ele.get('source').get('cell')))}.{field_dict.get(int(ele.get('source').get('port')))}={table_dict.get(int(ele.get('target').get('cell')))}.{field_dict.get(int(ele.get('target').get('port')))}\n" + schema_str += '【Foreign keys】\n' + for source, target in relations: + schema_str += (f"{eligible[source[0]]['table_name']}.{visible_fields[source]}=" + f"{eligible[target[0]]['table_name']}.{visible_fields[target]}\n") return schema_str, table_name_list diff --git a/backend/apps/datasource/crud/embedding_state.py b/backend/apps/datasource/crud/embedding_state.py new file mode 100644 index 00000000..cc9f159d --- /dev/null +++ b/backend/apps/datasource/crud/embedding_state.py @@ -0,0 +1,19 @@ +"""Atomic metadata invalidation. Lock parents before children in every writer.""" + +from sqlalchemy import update + +from apps.datasource.models.datasource import CoreDatasource, CoreTable + + +def invalidate_metadata(session, ds_id: int, table_ids=()): + # These row locks also serialize the short validation/publication phase of + # generation. No business row is locked while the model provider is running. + session.execute( + update(CoreDatasource).where(CoreDatasource.id == ds_id).values(embedding=None) + ) + if table_ids: + session.execute( + update(CoreTable) + .where(CoreTable.ds_id == ds_id, CoreTable.id.in_(table_ids)) + .values(embedding=None) + ) diff --git a/backend/apps/datasource/crud/field.py b/backend/apps/datasource/crud/field.py index ad6ee750..5ada0cdc 100644 --- a/backend/apps/datasource/crud/field.py +++ b/backend/apps/datasource/crud/field.py @@ -1,3 +1,4 @@ +from .embedding_state import invalidate_metadata from common.core.deps import SessionDep from ..models.datasource import CoreField, FieldObj, CoreTable from sqlalchemy import or_, and_ @@ -26,6 +27,7 @@ def get_fields_by_table_id(session: SessionDep, ds_id: int, id: int, field: Fiel def update_field(session: SessionDep, item: CoreField): record = session.query(CoreField).filter(CoreField.id == item.id).first() + invalidate_metadata(session, record.ds_id, [record.table_id]) record.checked = item.checked record.custom_comment = item.custom_comment session.add(record) diff --git a/backend/apps/datasource/crud/table.py b/backend/apps/datasource/crud/table.py index d5685e57..a8256c78 100644 --- a/backend/apps/datasource/crud/table.py +++ b/backend/apps/datasource/crud/table.py @@ -1,161 +1,191 @@ import json -import time -import traceback -from typing import List -from sqlalchemy import and_, select, update +from sqlalchemy import select, update from apps.ai_model.embedding import EmbeddingModelCache -from common.core.config import settings +from apps.datasource.embedding.utils import parse_valid_embedding from common.core.deps import SessionDep +from common.core.embedding_config import load_embedding_config +from common.utils.embedding_lifecycle import ( + EmbeddingGenerationError, + lock_embedding_resource, +) from common.utils.utils import SQLBotLogUtil -from ..models.datasource import CoreTable, CoreField, CoreDatasource + +from ..models.datasource import CoreDatasource, CoreField, CoreTable +from .embedding_state import invalidate_metadata def delete_table_by_ds_id(session: SessionDep, id: int): - session.query(CoreTable).filter(CoreTable.ds_id == id).delete(synchronize_session=False) + session.query(CoreTable).filter(CoreTable.ds_id == id).delete( + synchronize_session=False + ) session.commit() def get_tables_by_ds_id(session: SessionDep, id: int): - return session.query(CoreTable).filter(CoreTable.ds_id == id).order_by( - CoreTable.table_name.asc()).all() + return ( + session.query(CoreTable) + .filter(CoreTable.ds_id == id) + .order_by(CoreTable.table_name.asc()) + .all() + ) def update_table(session: SessionDep, item: CoreTable): record = session.query(CoreTable).filter(CoreTable.id == item.id).first() + invalidate_metadata(session, record.ds_id, [record.id]) record.checked = item.checked record.custom_comment = item.custom_comment session.add(record) session.commit() -def run_fill_empty_table_and_ds_embedding(session_maker): +def _missing(vector): try: - if not settings.TABLE_EMBEDDING_ENABLED: + parse_valid_embedding(vector) + return False + except (TypeError, ValueError): + return True + + +def _table_input(session, table): + comment = (table.custom_comment or table.table_comment or "").strip() + result = ( + f"# Table: {table.table_name}" + (f", {comment}" if comment else "") + "\n[\n" + ) + fields = ( + session.query(CoreField) + .filter(CoreField.table_id == table.id, CoreField.checked.is_(True)) + .order_by(CoreField.field_index, CoreField.id) + .all() + ) + lines = [] + for field in fields: + comment = (field.custom_comment or field.field_comment or "").strip() + lines.append( + f"({field.field_name}:{field.field_type}" + + (f", {comment}" if comment else "") + + ")" + ) + return result + ",\n".join(lines) + "\n]\n" + + +def _resource_input(session, record): + if isinstance(record, CoreTable): + return _table_input(session, record) + tables = ( + session.query(CoreTable) + .filter(CoreTable.ds_id == record.id, CoreTable.checked.is_(True)) + .order_by(CoreTable.id) + .all() + ) + return f"{record.name}, {record.description}\n" + "".join( + _table_input(session, table) for table in tables + ) + + +def _generate(session_maker, resource, ids=None, config=None): + session = session_maker() + failed = [] + try: + config = config or load_embedding_config(session) + if not config.table_enabled: return - - session = session_maker() - - SQLBotLogUtil.info('get tables') - stmt = select(CoreTable.id).where(and_(CoreTable.embedding.is_(None))) - results = session.execute(stmt).scalars().all() - SQLBotLogUtil.info('table result: ' + str(len(results))) - save_table_embedding(session_maker, results) - - SQLBotLogUtil.info('get datasource') - ds_stmt = select(CoreDatasource.id).where(and_(CoreDatasource.embedding.is_(None))) - ds_results = session.execute(ds_stmt).scalars().all() - SQLBotLogUtil.info('datasource result: ' + str(len(ds_results))) - save_ds_embedding(session_maker, ds_results) - except Exception: - traceback.print_exc() + query = select(resource.id, resource.embedding).order_by(resource.id) + if ids is not None: + query = query.where(resource.id.in_(ids)) + pending = [key for key, vector in session.execute(query) if _missing(vector)] + session.commit() + for key in pending: + for attempt in range(3): + try: + lock_embedding_resource(session, resource.__tablename__, key) + session.expire_all() + record = session.get(resource, key) + if record is None or not _missing(record.embedding): + session.commit() + break + original = _resource_input(session, record) + vector = parse_valid_embedding( + EmbeddingModelCache.get_model().embed_query(original) + ) + # Writers invalidate parent first, then child, in the same + # transaction as metadata. Match that order for publication. + # The provider call above holds no business row locks. + ds_id = record.ds_id if resource is CoreTable else key + parent = session.execute( + select(CoreDatasource.id) + .where(CoreDatasource.id == ds_id) + .with_for_update() + ).scalar_one_or_none() + if parent is None: + session.rollback() + break + if resource is CoreTable: + session.execute( + select(CoreTable.id) + .where(CoreTable.id == key) + .with_for_update() + ).all() + session.expire_all() + current = session.get(resource, key) + if current is None: + session.rollback() + break + if original != _resource_input(session, current): + session.rollback() + if attempt == 2: + failed.append(key) + continue + if _missing(current.embedding): + session.execute( + update(resource) + .where(resource.id == key) + .values( + embedding=json.dumps([float(value) for value in vector]) + ) + ) + session.commit() + break + except Exception: + session.rollback() + if attempt == 2: + failed.append(key) + SQLBotLogUtil.exception( + "Embedding remains pending for %s: %s", + resource.__tablename__, + key, + ) + if failed: + raise EmbeddingGenerationError(resource.__tablename__, failed) finally: session_maker.remove() -def save_table_embedding(session_maker, ids: List[int]): - if not settings.TABLE_EMBEDDING_ENABLED: - return - - if not ids or len(ids) == 0: - return +def run_fill_empty_table_and_ds_embedding(session_maker): + # Each startup/reenable is an independent recoverable scan. Resource locks + # deduplicate work across workers and are released even when a process dies. + session = session_maker() try: - SQLBotLogUtil.info('start table embedding') - start_time = time.time() - model = EmbeddingModelCache.get_model() - session = session_maker() - for _id in ids: - table = session.query(CoreTable).filter(CoreTable.id == _id).first() - fields = session.query(CoreField).filter(CoreField.table_id == table.id).all() - - schema_table = '' - schema_table += f"# Table: {table.table_name}" - table_comment = '' - if table.custom_comment: - table_comment = table.custom_comment.strip() - if table_comment == '': - schema_table += '\n[\n' - else: - schema_table += f", {table_comment}\n[\n" - - if fields: - field_list = [] - for field in fields: - field_comment = '' - if field.custom_comment: - field_comment = field.custom_comment.strip() - if field_comment == '': - field_list.append(f"({field.field_name}:{field.field_type})") - else: - field_list.append(f"({field.field_name}:{field.field_type}, {field_comment})") - schema_table += ",\n".join(field_list) - schema_table += '\n]\n' - # table_schema.append(schema_table) - emb = json.dumps(model.embed_query(schema_table)) - - stmt = update(CoreTable).where(and_(CoreTable.id == _id)).values(embedding=emb) - session.execute(stmt) - session.commit() - - end_time = time.time() - SQLBotLogUtil.info('table embedding finished in: ' + str(end_time - start_time) + ' seconds') - except Exception: - traceback.print_exc() + config = load_embedding_config(session) finally: session_maker.remove() + if not config.table_enabled: + return + failures = [] + for resource in (CoreTable, CoreDatasource): + try: + _generate(session_maker, resource, config=config) + except EmbeddingGenerationError as exc: + failures.extend(exc.ids) + if failures: + raise EmbeddingGenerationError("datasource_metadata", failures) -def save_ds_embedding(session_maker, ids: List[int]): - if not settings.TABLE_EMBEDDING_ENABLED: - return +def save_table_embedding(session_maker, ids: list[int]): + _generate(session_maker, CoreTable, ids) - if not ids or len(ids) == 0: - return - try: - SQLBotLogUtil.info('start datasource embedding') - start_time = time.time() - model = EmbeddingModelCache.get_model() - session = session_maker() - for _id in ids: - schema_table = '' - ds = session.query(CoreDatasource).filter(CoreDatasource.id == _id).first() - schema_table += f"{ds.name}, {ds.description}\n" - tables = session.query(CoreTable).filter(CoreTable.ds_id == ds.id).all() - for table in tables: - fields = session.query(CoreField).filter(CoreField.table_id == table.id).all() - - schema_table += f"# Table: {table.table_name}" - table_comment = '' - if table.custom_comment: - table_comment = table.custom_comment.strip() - if table_comment == '': - schema_table += '\n[\n' - else: - schema_table += f", {table_comment}\n[\n" - - if fields: - field_list = [] - for field in fields: - field_comment = '' - if field.custom_comment: - field_comment = field.custom_comment.strip() - if field_comment == '': - field_list.append(f"({field.field_name}:{field.field_type})") - else: - field_list.append(f"({field.field_name}:{field.field_type}, {field_comment})") - schema_table += ",\n".join(field_list) - schema_table += '\n]\n' - # table_schema.append(schema_table) - emb = json.dumps(model.embed_query(schema_table)) - - stmt = update(CoreDatasource).where(and_(CoreDatasource.id == _id)).values(embedding=emb) - session.execute(stmt) - session.commit() - - end_time = time.time() - SQLBotLogUtil.info('datasource embedding finished in: ' + str(end_time - start_time) + ' seconds') - except Exception: - traceback.print_exc() - finally: - session_maker.remove() + +def save_ds_embedding(session_maker, ids: list[int]): + _generate(session_maker, CoreDatasource, ids) diff --git a/backend/apps/datasource/embedding/ds_embedding.py b/backend/apps/datasource/embedding/ds_embedding.py index f75ee7df..d40478f8 100644 --- a/backend/apps/datasource/embedding/ds_embedding.py +++ b/backend/apps/datasource/embedding/ds_embedding.py @@ -1,91 +1,60 @@ -# Author: Junjun -# Date: 2025/9/18 -import json -import time +"""Vector preselection of already-authorized datasource candidates.""" + import traceback -from typing import Optional from apps.ai_model.embedding import EmbeddingModelCache -from apps.datasource.embedding.utils import cosine_similarity +from apps.datasource.embedding.utils import cosine_similarity, parse_valid_embedding from apps.datasource.models.datasource import CoreDatasource from apps.system.crud.assistant import AssistantOutDs -from common.core.config import settings -from common.core.deps import CurrentAssistant -from common.core.deps import SessionDep +from common.core.deps import CurrentAssistant, SessionDep +from common.core.embedding_config import EmbeddingConfig, load_embedding_config from common.utils.utils import SQLBotLogUtil -def get_ds_embedding(session: SessionDep, _ds_list, out_ds: AssistantOutDs, - question: str, - current_assistant: Optional[CurrentAssistant] = None): - _list = [] - if current_assistant and current_assistant.type == 1: - if out_ds.ds_list: - for _ds in out_ds.ds_list: - ds = out_ds.get_ds(_ds.id) - table_schema, tables = out_ds.get_db_schema(_ds.id, question, embedding=False) - ds_info = f"{ds.name}, {ds.description}\n" - ds_schema = ds_info + table_schema - _list.append({"id": ds.id, "ds_schema": ds_schema, "cosine_similarity": 0.0, "ds": ds}) - - if _list: - try: - text = [s.get('ds_schema') for s in _list] - - model = EmbeddingModelCache.get_model() - results = model.embed_documents(text) - - q_embedding = model.embed_query(question) - for index in range(len(results)): - item = results[index] - _list[index]['cosine_similarity'] = cosine_similarity(q_embedding, item) - - _list.sort(key=lambda x: x['cosine_similarity'], reverse=True) - # print(len(_list)) - _list = _list[:settings.DS_EMBEDDING_COUNT] - SQLBotLogUtil.info(json.dumps( - [{"id": ele.get("id"), "name": ele.get("ds").name, - "cosine_similarity": ele.get("cosine_similarity")} - for ele in _list])) - return [{"id": obj.get('ds').id, "name": obj.get('ds').name, "description": obj.get('ds').description} - for obj in _list] - except Exception: - traceback.print_exc() - else: - for _ds in _ds_list: - if _ds.get('id'): - ds = session.get(CoreDatasource, _ds.get('id')) - # table_schema = get_table_schema(session, current_user, ds, question, embedding=False) - # ds_info = f"{ds.name}, {ds.description}\n" - # ds_schema = ds_info + table_schema - _list.append({"id": ds.id, "cosine_similarity": 0.0, "ds": ds, "embedding": ds.embedding}) - - if _list: - try: - # text = [s.get('ds_schema') for s in _list] - - model = EmbeddingModelCache.get_model() - start_time = time.time() - # results = model.embed_documents(text) - results = [item.get('embedding') for item in _list] - - q_embedding = model.embed_query(question) - for index in range(len(results)): - item = results[index] - if item: - _list[index]['cosine_similarity'] = cosine_similarity(q_embedding, json.loads(item)) - - _list.sort(key=lambda x: x['cosine_similarity'], reverse=True) - # print(len(_list)) - end_time = time.time() - SQLBotLogUtil.info(str(end_time - start_time)) - _list = _list[:settings.DS_EMBEDDING_COUNT] - SQLBotLogUtil.info(json.dumps( - [{"id": ele.get("id"), "name": ele.get("ds").name, - "cosine_similarity": ele.get("cosine_similarity")} - for ele in _list])) - return [{"id": obj.get('ds').id, "name": obj.get('ds').name, "description": obj.get('ds').description} - for obj in _list] - except Exception: - traceback.print_exc() - return _list +def get_ds_embedding( + session: SessionDep, + _ds_list, + out_ds: AssistantOutDs, # noqa: ARG001 - retained for existing callers + question: str, + current_assistant: CurrentAssistant | None = None, + *, + config: EmbeddingConfig | None = None, +): + # Advanced applications keep their existing model-only datasource selection. + if current_assistant and current_assistant.type in (1, 3): + return _ds_list + config = config or load_embedding_config(session) + if not config.table_enabled or len(_ds_list) <= 1: + return _ds_list + try: + vectors = [] + for candidate in _ds_list: + datasource = session.get(CoreDatasource, candidate.get("id")) + vectors.append( + parse_valid_embedding(datasource.embedding if datasource else None) + ) + query = parse_valid_embedding( + EmbeddingModelCache.get_model().embed_query(question) + ) + ranked = sorted( + zip(_ds_list, vectors, strict=True), + key=lambda item: cosine_similarity(query, item[1]), + reverse=True, + ) + selected = [candidate for candidate, _vector in ranked[: config.datasource_count]] + SQLBotLogUtil.info( + f"[embedding] stage=datasource action=vector_selection " + f"candidate_count={len(_ds_list)} result_count={len(selected)} " + f"limit={config.datasource_count}" + ) + return selected + except Exception as exc: + traceback.print_exc() + SQLBotLogUtil.info( + f"[embedding] stage=datasource action=fallback " + f"reason=invalid_vectors_or_scoring_failed error_type={type(exc).__name__} " + f"candidate_count={len(_ds_list)} result_count={len(_ds_list)}" + ) + # Incomplete/stale vectors or a failed model must not remove candidates, + # nor return ORM/internal fields in the model-facing candidate DTOs. + return _ds_list diff --git a/backend/apps/datasource/embedding/table_embedding.py b/backend/apps/datasource/embedding/table_embedding.py index b4d95ec9..87e50307 100644 --- a/backend/apps/datasource/embedding/table_embedding.py +++ b/backend/apps/datasource/embedding/table_embedding.py @@ -6,8 +6,13 @@ import traceback from apps.ai_model.embedding import EmbeddingModelCache -from apps.datasource.embedding.utils import batch_cosine_similarity, cosine_similarity +from apps.datasource.embedding.utils import ( + batch_cosine_similarity, + cosine_similarity, + parse_valid_embedding, +) from common.core.config import settings +from common.core.embedding_config import EmbeddingConfig, load_embedding_config from common.utils.utils import SQLBotLogUtil # 预编译正则，避免每次调用都重新编译 @@ -36,7 +41,7 @@ def _longest_mentions(spans_by_name: dict): if any(span in longest_spans for span in occurrences)} -def _select_tables(ranked_tables: list[dict], question: str, *, apply_limit: bool = True): +def _select_tables(ranked_tables: list[dict], question: str, *, apply_limit: bool = True, count: int = None): """Keep explicitly mentioned candidates, then fill remaining slots by existing rank. Inspect only candidates already filtered by the caller; retain duplicate comments. @@ -73,19 +78,10 @@ def _select_tables(ranked_tables: list[dict], question: str, *, apply_limit: boo else: remaining.append(table) if apply_limit: - remaining = remaining[:max(0, settings.TABLE_EMBEDDING_COUNT - len(required))] + remaining = remaining[:max(0, (settings.TABLE_EMBEDDING_COUNT if count is None else count) - len(required))] return required + remaining -def _parse_embedding(embedding): - """解析 embedding，兼容 JSON 字符串和原生列表两种格式。""" - if isinstance(embedding, list): - return embedding - if isinstance(embedding, str): - return json.loads(embedding) - return None - - def get_table_embedding(tables: list[dict], question: str): _list = [] for table in tables: @@ -232,8 +228,17 @@ def calc_keyword_score( return round(best_score * match_rate, 2) +def _score_table_vectors(tables: list[dict], query_embedding) -> int: + """Validate the complete candidate set before publishing any vector scores.""" + embeddings = [parse_valid_embedding(table.get('embedding')) for table in tables] + similarities = batch_cosine_similarity(parse_valid_embedding(query_embedding), embeddings) + for table, similarity in zip(tables, similarities, strict=True): + table['cosine_similarity'] = float(similarity) + return len(embeddings) + + def calc_table_embedding(tables: list[dict], question: str, session=None, oid: int = None, - keywords: str = None): + keywords: str = None, *, config: EmbeddingConfig = None): """使用向量相似度和关键词匹配的融合评分计算表的相关性分数。 当 keywords 不为空时（已由调用方提取并扩展）： @@ -251,9 +256,11 @@ def calc_table_embedding(tables: list[dict], question: str, session=None, oid: i oid: 组织 ID（预留） keywords: 已提取并扩展的关键词（由调用方传入） """ - if not keywords or not settings.TABLE_EMBEDDING_KEYWORD_ENABLED: + if config is None and session is not None: + config = load_embedding_config(session) + if not keywords or not (config.keyword_enabled if config else settings.TABLE_EMBEDDING_KEYWORD_ENABLED): # 回退到纯向量匹配 - return _calc_vector_only(tables, question) + return _calc_vector_only(tables, question, config=config) _list = [] for table in tables: @@ -280,28 +287,14 @@ def calc_table_embedding(tables: list[dict], question: str, session=None, oid: i t1 = time.time() SQLBotLogUtil.info(f"[perf] embed_query 耗时 {t1 - t0:.3f}s") - # 预解析所有 embedding，跳过空的 - parsed_embeddings = [] - valid_indices = [] - for i, item in enumerate(_list): - emb = item.get('embedding') - if emb: - parsed = _parse_embedding(emb) - if parsed: - parsed_embeddings.append(parsed) - valid_indices.append(i) - - # 一次性批量计算所有余弦相似度 - if parsed_embeddings: - similarities = batch_cosine_similarity(q_embedding, parsed_embeddings) - for idx, sim in zip(valid_indices, similarities): - _list[idx]['cosine_similarity'] = float(sim) + # A partial vector set must never exclude otherwise eligible tables. + vector_count = _score_table_vectors(_list, q_embedding) t2 = time.time() - SQLBotLogUtil.info(f"[perf] 向量相似度计算耗时 {t2 - t1:.3f}s，共 {len(parsed_embeddings)} 张表") + SQLBotLogUtil.info(f"[perf] 向量相似度计算耗时 {t2 - t1:.3f}s，共 {vector_count} 张表") # 步骤 2 & 3：计算关键词分数并融合 - alpha = settings.TABLE_EMBEDDING_ALPHA + alpha = config.table_alpha if config else settings.TABLE_EMBEDDING_ALPHA kw_total = 0.0 kw_max = 0.0 kw_max_name = '' @@ -335,7 +328,7 @@ def calc_table_embedding(tables: list[dict], question: str, session=None, oid: i other_tables.sort(key=lambda x: x['cosine_similarity'], reverse=True) _list = exact_matches + other_tables - _list = _select_tables(_list, question) + _list = _select_tables(_list, question, count=config.table_count if config else None) end_time = time.time() SQLBotLogUtil.info(f"[perf] 融合评分总耗时 {end_time - start_time:.3f}s") @@ -348,13 +341,18 @@ def calc_table_embedding(tables: list[dict], question: str, session=None, oid: i } for ele in _list])) return _list - except Exception: + except Exception as exc: traceback.print_exc() - # 异常回退到纯向量匹配 - return _calc_vector_only(tables, question) + # Skip this preselection layer if vectors are incomplete or scoring fails. + SQLBotLogUtil.info( + f"[embedding] stage=table action=fallback mode=hybrid " + f"reason=invalid_vectors_or_scoring_failed error_type={type(exc).__name__} " + f"candidate_count={len(tables)} result_count={len(tables)}" + ) + return _select_tables(tables, question, apply_limit=False) -def _calc_vector_only(tables: list[dict], question: str): +def _calc_vector_only(tables: list[dict], question: str, *, config: EmbeddingConfig = None): """Rank by vector similarity while retaining explicitly mentioned tables.""" _list = [] for table in tables: @@ -374,28 +372,13 @@ def _calc_vector_only(tables: list[dict], question: str): start_time = time.time() q_embedding = model.embed_query(question) - # 预解析所有 embedding，跳过空的 - parsed_embeddings = [] - valid_indices = [] - for i, item in enumerate(_list): - emb = item.get('embedding') - if emb: - parsed = _parse_embedding(emb) - if parsed: - parsed_embeddings.append(parsed) - valid_indices.append(i) - - # 一次性批量计算所有余弦相似度 - if parsed_embeddings: - similarities = batch_cosine_similarity(q_embedding, parsed_embeddings) - for idx, sim in zip(valid_indices, similarities): - _list[idx]['cosine_similarity'] = float(sim) + vector_count = _score_table_vectors(_list, q_embedding) _list.sort(key=lambda x: x['cosine_similarity'], reverse=True) - _list = _select_tables(_list, question) + _list = _select_tables(_list, question, count=config.table_count if config else None) end_time = time.time() - SQLBotLogUtil.info(f"[perf] 纯向量匹配耗时 {end_time - start_time:.3f}s，共 {len(parsed_embeddings)} 张表") + SQLBotLogUtil.info(f"[perf] 纯向量匹配耗时 {end_time - start_time:.3f}s，共 {vector_count} 张表") SQLBotLogUtil.info(json.dumps([{ "id": ele.get('id'), "schema_table": ele.get('schema_table'), @@ -403,7 +386,12 @@ def _calc_vector_only(tables: list[dict], question: str): "table_name": ele.get('table_name') } for ele in _list])) return _list - except Exception: + except Exception as exc: traceback.print_exc() + SQLBotLogUtil.info( + f"[embedding] stage=table action=fallback mode=vector_only " + f"reason=invalid_vectors_or_scoring_failed error_type={type(exc).__name__} " + f"candidate_count={len(_list)} result_count={len(_list)}" + ) # Preserve all candidates when vector ranking fails; move explicit matches to the front. return _select_tables(_list, question, apply_limit=False) diff --git a/backend/apps/datasource/embedding/utils.py b/backend/apps/datasource/embedding/utils.py index 4eefeae3..e47aaaf3 100644 --- a/backend/apps/datasource/embedding/utils.py +++ b/backend/apps/datasource/embedding/utils.py @@ -1,5 +1,6 @@ # Author: Junjun # Date: 2025/9/23 +import json import math import numpy as np @@ -44,3 +45,15 @@ def batch_cosine_similarity(query_vec, table_vecs): norms[norms == 0] = 1.0 # 零向量相似度为 0，除以 1 保持结果为 0 return mat @ q / (norms * q_norm) + + +def parse_valid_embedding(value): + """Reject absent, malformed or unusable JSON vectors before candidate pruning.""" + vector = json.loads(value) if isinstance(value, str) else value + if not isinstance(vector, list) or not vector: + raise ValueError("Missing embedding") + if any(type(number) not in (int, float) or not math.isfinite(number) for number in vector): + raise ValueError("Invalid embedding") + if not any(vector): + raise ValueError("Zero embedding") + return vector diff --git a/backend/apps/system/crud/parameter_manage.py b/backend/apps/system/crud/parameter_manage.py index 65aaf164..25cd94e1 100644 --- a/backend/apps/system/crud/parameter_manage.py +++ b/backend/apps/system/crud/parameter_manage.py @@ -1,18 +1,39 @@ +import asyncio +import json + from fastapi import Request +from sqlalchemy import text from sqlbot_xpack.config.arg_manage import get_group_args, save_group_args from sqlbot_xpack.config.model import SysArgModel -import json -from common.core.deps import SessionDep from sqlbot_xpack.file_utils import SQLBotFileUtils +from common.core.deps import SessionDep +from common.core.embedding_config import ( + EDITABLE_FIELDS, + effective_parameter_args, + load_embedding_config, + parse_parameter, +) +from common.error import SingleMessageError +from common.utils.locale import I18n + +# Fixed bigint advisory-lock key for all parameter writers. Resource generation +# keys instead hash the sqlbot:embedding:: namespace. +PARAMETER_SAVE_LOCK_KEY = 14030001 + + async def get_parameter_args(session: SessionDep) -> list[SysArgModel]: group_args = await get_group_args(session=session) - return [x for x in group_args if not x.pkey.startswith('appearance.')] + return [ + x for x in group_args if not x.pkey.startswith(("appearance.", "embedding.")) + ] + effective_parameter_args(session) + async def get_groups(session: SessionDep, flag: str) -> list[SysArgModel]: group_args = await get_group_args(session=session, flag=flag) return group_args + async def save_parameter_args(session: SessionDep, request: Request): allow_file_mapping = { """ "test_logo": { "types": [".jpg", ".jpeg", ".png"], "size": 5 * 1024 * 1024 } """ @@ -20,11 +41,49 @@ async def save_parameter_args(session: SessionDep, request: Request): form_data = await request.form() files = form_data.getlist("files") json_text = form_data.get("data") - sys_args = [ - SysArgModel(**{**item, "pkey": f"{item['pkey']}"}) - for item in json.loads(json_text) - if "pkey" in item - ] + # xpack commits internally: validate the complete batch before passing any values. + try: + items = json.loads(json_text) + except (TypeError, json.JSONDecodeError): + raise SingleMessageError( + I18n()(request)("i18n_parameter.invalid_embedding_parameter", key="data") + ) + if not isinstance(items, list) or any(not isinstance(item, dict) for item in items): + raise SingleMessageError( + I18n()(request)("i18n_parameter.invalid_embedding_parameter", key="data") + ) + sys_args = [] + seen = set() + for item in items: + key = item.get("pkey") + if not isinstance(key, str) or not key: + raise SingleMessageError( + I18n()(request)( + "i18n_parameter.invalid_embedding_parameter", key="pkey" + ) + ) + if key.startswith("embedding."): + field = key.removeprefix("embedding.") + try: + if field not in EDITABLE_FIELDS or key in seen: + raise ValueError(key) + value = parse_parameter(field, item.get("pval")) + except (ValueError, TypeError, OverflowError): + raise SingleMessageError( + I18n()(request)( + "i18n_parameter.invalid_embedding_parameter", key=key + ) + ) + item = {**item, "pval": str(value).lower()} + else: + item = { + **item, + "pval": str(item.get("pval", "")).lower() + if isinstance(item.get("pval"), bool) + else str(item.get("pval", "")), + } + seen.add(key) + sys_args.append(SysArgModel(**item)) if not sys_args: return file_mapping = None @@ -32,14 +91,49 @@ async def save_parameter_args(session: SessionDep, request: Request): file_mapping = {} for file in files: origin_file_name = file.filename - file_name, flag_name = SQLBotFileUtils.split_filename_and_flag(origin_file_name) + file_name, flag_name = SQLBotFileUtils.split_filename_and_flag( + origin_file_name + ) file.filename = file_name allow_limit_obj = allow_file_mapping.get(flag_name) if allow_limit_obj: - SQLBotFileUtils.check_file(file=file, file_types=allow_limit_obj.get("types"), limit_file_size=allow_limit_obj.get("size")) + SQLBotFileUtils.check_file( + file=file, + file_types=allow_limit_obj.get("types"), + limit_file_size=allow_limit_obj.get("size"), + ) else: - raise Exception(f'The file [{file_name}] is not allowed to be uploaded!') + raise Exception( + f"The file [{file_name}] is not allowed to be uploaded!" + ) file_id = await SQLBotFileUtils.upload(file) file_mapping[f"{flag_name}"] = file_id - - await save_group_args(session=session, sys_args=sys_args, file_mapping=file_mapping) \ No newline at end of file + + # Legacy migrations do not enforce unique pkey. Serialize first-time writes + # across workers before the adapter reads existing keys and commits. A try + # lock lets other requests run while waiting. Keep the async adapter on its + # original loop so configured cache clients retain their loop affinity. + if session.get_bind().dialect.name == "postgresql": + while not session.execute( + text("SELECT pg_try_advisory_xact_lock(:key)"), + {"key": PARAMETER_SAVE_LOCK_KEY}, + ).scalar_one(): + await asyncio.sleep(0.05) + previous = load_embedding_config(session) + result = await save_group_args( + session=session, sys_args=sys_args, file_mapping=file_mapping + ) + current = load_embedding_config(session) + if not previous.enabled and current.enabled: + from common.utils.embedding_threads import ( + fill_empty_data_training_embeddings, + fill_empty_terminology_embeddings, + ) + + fill_empty_terminology_embeddings() + fill_empty_data_training_embeddings() + if not previous.table_enabled and current.table_enabled: + from common.utils.embedding_threads import fill_empty_table_and_ds_embeddings + + fill_empty_table_and_ds_embeddings() + return result diff --git a/backend/apps/terminology/curd/terminology.py b/backend/apps/terminology/curd/terminology.py index a3864bd7..0cb50548 100644 --- a/backend/apps/terminology/curd/terminology.py +++ b/backend/apps/terminology/curd/terminology.py @@ -3,17 +3,19 @@ from typing import List, Optional, Any from xml.dom.minidom import parseString -from sqlalchemy import and_, or_, select, func, delete, update, union, text, BigInteger +from sqlalchemy import and_, or_, select, func, delete, update, text, BigInteger, case from sqlalchemy.orm import aliased from apps.ai_model.embedding import EmbeddingModelCache +from apps.datasource.embedding.utils import parse_valid_embedding from apps.datasource.models.datasource import CoreDatasource from apps.system.models.system_model import AssistantModel from apps.template.generate_chart.generator import get_base_terminology_template from apps.terminology.models.terminology_model import Terminology, TerminologyInfo, TerminologyInfoResult -from common.core.config import settings +from common.core.embedding_config import EmbeddingConfig, load_embedding_config from common.core.deps import SessionDep, Trans from common.utils.dict_to_xml import dict_to_xml +from common.utils.embedding_lifecycle import generate_missing_embeddings from common.utils.embedding_threads import run_save_terminology_embeddings @@ -372,7 +374,8 @@ def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int, tra oid=oid, enabled=info.enabled, specific_ds=specific_ds, - datasource_ids=datasource_ids + datasource_ids=datasource_ids, + advanced_application=info.advanced_application, ) ) @@ -591,8 +594,9 @@ def update_terminology(session: SessionDep, info: TerminologyInfo, oid: int, tra if not datasource_ids and info.advanced_application is None: raise Exception(trans("i18n_data_training.datasource_assistant_cannot_be_none")) + other_words = info.other_words or [] words = [info.word.strip()] - for child in info.other_words: + for child in other_words: if child in words: raise Exception(trans("i18n_terminology.cannot_be_repeated")) else: @@ -671,6 +675,7 @@ def update_terminology(session: SessionDep, info: TerminologyInfo, oid: int, tra raise Exception(trans("i18n_terminology.exists_in_db")) stmt = update(Terminology).where(and_(Terminology.id == info.id)).values( + embedding=case((Terminology.word.is_distinct_from(info.word.strip()), None), else_=Terminology.embedding), word=info.word.strip(), description=info.description.strip(), specific_ds=specific_ds, @@ -678,36 +683,37 @@ def update_terminology(session: SessionDep, info: TerminologyInfo, oid: int, tra enabled=info.enabled, advanced_application=info.advanced_application, ) - session.execute(stmt) - session.commit() - - stmt = delete(Terminology).where(and_(Terminology.pid == info.id)) - session.execute(stmt) - session.commit() - - create_time = datetime.datetime.now() - # 插入子记录（其他词） - child_list = [] - if info.other_words: - for other_word in info.other_words: - if other_word.strip() == "": - continue - child_list.append( - Terminology( - pid=info.id, - word=other_word.strip(), - create_time=create_time, - oid=oid, - enabled=info.enabled, - specific_ds=specific_ds, - datasource_ids=datasource_ids, - advanced_application=info.advanced_application, - ) - ) - - if child_list: - session.bulk_save_objects(child_list) - session.flush() + # The parent update serializes edits of this term. Keep input invalidation, + # synonym removal/addition and scope propagation in one transaction. + if session.execute(stmt).rowcount != 1: + raise Exception(trans('i18n_terminology.terminology_not_exists')) + desired_words = {word.strip() for word in other_words if word.strip()} + children = session.execute( + select(Terminology.id, Terminology.word).where(Terminology.pid == info.id) + ).all() + existing_words = set() + for child_id, word in children: + if word not in desired_words: + session.execute(delete(Terminology).where(Terminology.id == child_id)) + else: + existing_words.add(word) + session.execute(update(Terminology).where(Terminology.id == child_id).values( + enabled=info.enabled, + specific_ds=specific_ds, + datasource_ids=datasource_ids, + advanced_application=info.advanced_application, + )) + for word in sorted(desired_words - existing_words): + session.add(Terminology( + pid=info.id, + word=word, + create_time=datetime.datetime.now(), + oid=oid, + enabled=info.enabled, + specific_ds=specific_ds, + datasource_ids=datasource_ids, + advanced_application=info.advanced_application, + )) session.commit() # embedding @@ -739,108 +745,66 @@ def enable_terminology(session: SessionDep, id: int, enabled: bool, trans: Trans session.commit() -# def run_save_embeddings(ids: List[int]): -# executor.submit(save_embeddings, ids) -# -# -# def fill_empty_embeddings(): -# executor.submit(run_fill_empty_embeddings) -# from sqlalchemy import create_engine -# from sqlalchemy.orm import sessionmaker,scoped_session -# engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) -# session_maker = scoped_session(sessionmaker(bind=engine)) - def run_fill_empty_embeddings(session_maker): - try: - if not settings.EMBEDDING_ENABLED: - return - session = session_maker() - stmt1 = select(Terminology.id).where(and_(Terminology.embedding.is_(None), Terminology.pid.is_(None))) - stmt2 = select(Terminology.pid).where( - and_(Terminology.embedding.is_(None), Terminology.pid.isnot(None))).distinct() - combined_stmt = union(stmt1, stmt2) - results = session.execute(combined_stmt).scalars().all() - save_embeddings(session_maker, results) - except Exception: - traceback.print_exc() - finally: - session_maker.remove() + return generate_missing_embeddings(session_maker, Terminology, Terminology.word) def save_embeddings(session_maker, ids: List[int]): - if not settings.EMBEDDING_ENABLED: - return - - if not ids or len(ids) == 0: + if not ids: return - try: - session = session_maker() - _list = session.query(Terminology).filter(or_(Terminology.id.in_(ids), Terminology.pid.in_(ids))).all() - - _words_list = [item.word for item in _list] - - model = EmbeddingModelCache.get_model() - - results = model.embed_documents(_words_list) - - for index in range(len(results)): - item = results[index] - stmt = update(Terminology).where(and_(Terminology.id == _list[index].id)).values(embedding=item) - session.execute(stmt) - session.commit() - - except Exception: - traceback.print_exc() - finally: - session_maker.remove() + return generate_missing_embeddings( + session_maker, Terminology, Terminology.word, + or_(Terminology.id.in_(ids), Terminology.pid.in_(ids)), + ) -embedding_sql = f""" +embedding_sql = """ SELECT id, pid, word, similarity FROM (SELECT id, pid, word, oid, specific_ds, datasource_ids, enabled, ( 1 - (embedding <=> :embedding_array) ) AS similarity FROM terminology AS child ) TEMP -WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid AND enabled = true +WHERE similarity > :similarity AND oid = :oid AND enabled = true AND (specific_ds = false OR specific_ds IS NULL) ORDER BY similarity DESC -LIMIT {settings.EMBEDDING_TERMINOLOGY_TOP_COUNT} +LIMIT :top_count """ -embedding_sql_with_datasource = f""" +embedding_sql_with_datasource = """ SELECT id, pid, word, similarity FROM (SELECT id, pid, word, oid, specific_ds, datasource_ids, enabled, ( 1 - (embedding <=> :embedding_array) ) AS similarity FROM terminology AS child ) TEMP -WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid AND enabled = true +WHERE similarity > :similarity AND oid = :oid AND enabled = true AND ( (specific_ds = false OR specific_ds IS NULL) OR (specific_ds = true AND datasource_ids IS NOT NULL AND datasource_ids @> jsonb_build_array(:datasource)) ) ORDER BY similarity DESC -LIMIT {settings.EMBEDDING_TERMINOLOGY_TOP_COUNT} +LIMIT :top_count """ -embedding_sql_with_advanced_application = f""" +embedding_sql_with_advanced_application = """ SELECT id, pid, word, similarity FROM (SELECT id, pid, word, oid, specific_ds, advanced_application, enabled, ( 1 - (embedding <=> :embedding_array) ) AS similarity FROM terminology AS child ) TEMP -WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} AND oid = :oid AND enabled = true +WHERE similarity > :similarity AND oid = :oid AND enabled = true AND advanced_application = :advanced_application_id ORDER BY similarity DESC -LIMIT {settings.EMBEDDING_TERMINOLOGY_TOP_COUNT} +LIMIT :top_count """ def select_terminology_by_word(session: SessionDep, word: str, oid: int, datasource: int = None, - advanced_application_id: Optional[int] = None): + advanced_application_id: Optional[int] = None, *, config: EmbeddingConfig | None = None): + config = config or load_embedding_config(session) if word.strip() == "": return [] @@ -883,24 +847,24 @@ def select_terminology_by_word(session: SessionDep, word: str, oid: int, datasou for row in results: _list.append(Terminology(id=row.id, word=row.word, pid=row.pid)) - if settings.EMBEDDING_ENABLED: + if config.enabled: with session.begin_nested(): try: model = EmbeddingModelCache.get_model() - embedding = model.embed_query(word) + embedding = parse_valid_embedding(model.embed_query(word)) if advanced_application_id is not None: results = session.execute(text(embedding_sql_with_advanced_application), - {'embedding_array': str(embedding), 'oid': oid, + {'embedding_array': str(embedding), 'oid': oid, 'similarity': config.terminology_similarity, 'top_count': config.terminology_top_count, 'advanced_application_id': advanced_application_id}).fetchall() elif datasource is not None: results = session.execute(text(embedding_sql_with_datasource), - {'embedding_array': str(embedding), 'oid': oid, + {'embedding_array': str(embedding), 'oid': oid, 'similarity': config.terminology_similarity, 'top_count': config.terminology_top_count, 'datasource': datasource}).fetchall() else: results = session.execute(text(embedding_sql), - {'embedding_array': str(embedding), 'oid': oid}).fetchall() + {'embedding_array': str(embedding), 'oid': oid, 'similarity': config.terminology_similarity, 'top_count': config.terminology_top_count}).fetchall() for row in results: _list.append(Terminology(id=row.id, word=row.word, pid=row.pid)) @@ -974,7 +938,9 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str: def get_terminology_template(session: SessionDep, question: str, oid: Optional[int] = 1, datasource: Optional[int] = None, - advanced_application_id: Optional[int] = None) -> tuple[str, list[dict]]: + advanced_application_id: Optional[int] = None, *, + config: EmbeddingConfig | None = None) -> tuple[str, list[dict]]: + config = config or load_embedding_config(session) if not oid: oid = 1 @@ -990,7 +956,7 @@ def get_terminology_template(session: SessionDep, question: str, oid: Optional[i seen_words = set() _results = [] for part in parts: - for item in select_terminology_by_word(session, part, oid, datasource, advanced_application_id): + for item in select_terminology_by_word(session, part, oid, datasource, advanced_application_id, config=config): item_words = set(item.get('words', [])) if isinstance(item, dict) else set() if item_words and not item_words.issubset(seen_words): seen_words.update(item_words) diff --git a/backend/common/core/embedding_config.py b/backend/common/core/embedding_config.py new file mode 100644 index 00000000..deee6eca --- /dev/null +++ b/backend/common/core/embedding_config.py @@ -0,0 +1,141 @@ +"""System-wide effective embedding strategy, captured once at each task boundary.""" + +from __future__ import annotations + +import math +import re +from dataclasses import dataclass, fields +from typing import TYPE_CHECKING + +from sqlmodel import Session, select + +from common.core.config import settings + +if TYPE_CHECKING: + from sqlbot_xpack.config.model import SysArgModel + + +@dataclass(frozen=True) +class EmbeddingConfig: + enabled: bool + terminology_similarity: float + terminology_top_count: int + data_training_similarity: float + data_training_top_count: int + table_enabled: bool + table_count: int + table_alpha: float + keyword_enabled: bool + datasource_count: int + + +# Stable persisted keys use the embedding group; startup settings remain deployment defaults. +PARAMETER_SETTINGS = { + "enabled": "EMBEDDING_ENABLED", + "terminology_similarity": "EMBEDDING_TERMINOLOGY_SIMILARITY", + "terminology_top_count": "EMBEDDING_TERMINOLOGY_TOP_COUNT", + "data_training_similarity": "EMBEDDING_DATA_TRAINING_SIMILARITY", + "data_training_top_count": "EMBEDDING_DATA_TRAINING_TOP_COUNT", + "table_enabled": "TABLE_EMBEDDING_ENABLED", + "table_count": "TABLE_EMBEDDING_COUNT", + "table_alpha": "TABLE_EMBEDDING_ALPHA", + "keyword_enabled": "TABLE_EMBEDDING_KEYWORD_ENABLED", + "datasource_count": "DS_EMBEDDING_COUNT", +} +EDITABLE_FIELDS = ( + "enabled", + "table_enabled", + "terminology_similarity", + "terminology_top_count", + "data_training_similarity", + "data_training_top_count", + "table_count", + "table_alpha", + "keyword_enabled", + "datasource_count", +) +BOOLEAN_FIELDS = {"enabled", "table_enabled", "keyword_enabled"} +INTEGER_FIELDS = { + "terminology_top_count", + "data_training_top_count", + "table_count", + "datasource_count", +} + + +def parse_parameter(field: str, value: object) -> bool | int | float: + """Strict saved-value parsing. In particular, bool is never an integer count.""" + if field in BOOLEAN_FIELDS: + if type(value) is bool: + return value + if isinstance(value, str) and value.lower() in {"true", "false"}: + return value.lower() == "true" + raise ValueError(field) + if type(value) is bool or value is None: + raise ValueError(field) + if field in INTEGER_FIELDS: + if not isinstance(value, (str, int)) or not re.fullmatch(r"[0-9]+", str(value)): + raise ValueError(field) + number = int(value) + if number <= 0: + raise ValueError(field) + return number + if not isinstance(value, (str, int, float)): + raise ValueError(field) + number = float(value) + if not math.isfinite(number) or not 0 <= number <= 1: + raise ValueError(field) + return number + + +def load_embedding_config(session: Session) -> EmbeddingConfig: + """Read all overrides in one statement; never mutate or cache startup settings. + + A new Session per task sees commits from any API worker or MCP process. Invalid + persisted overrides are rejected rather than silently replaced with defaults. + Migrated installations may not have a unique pkey constraint. For historical + duplicate keys, the row with the greatest id wins deterministically. + """ + # Released xpack imports datasource consumers during package bootstrap. + # Keep this model import lazy so cold worker imports cannot form a cycle. + from sqlbot_xpack.config.model import SysArgModel + + rows = ( + session.execute( + select(SysArgModel) + .where( + SysArgModel.pkey.in_( + [f"embedding.{name}" for name in PARAMETER_SETTINGS] + ) + ) + .order_by(SysArgModel.id) + ) + .scalars() + .all() + ) + overrides = {row.pkey.removeprefix("embedding."): row.pval for row in rows} + values = { + field.name: parse_parameter(field.name, overrides[field.name]) + if field.name in overrides + else getattr(settings, PARAMETER_SETTINGS[field.name]) + for field in fields(EmbeddingConfig) + } + return EmbeddingConfig(**values) + + +def effective_parameter_args(session: Session) -> list[SysArgModel]: + from sqlbot_xpack.config.model import SysArgModel + + config = load_embedding_config(session) + return [ + SysArgModel( + pkey=f"embedding.{name}", + pval=str(getattr(config, name)).lower(), + ptype="bool" + if name in BOOLEAN_FIELDS + else "int" + if name in INTEGER_FIELDS + else "float", + ) + for name in EDITABLE_FIELDS + ] diff --git a/backend/common/utils/embedding_lifecycle.py b/backend/common/utils/embedding_lifecycle.py new file mode 100644 index 00000000..c5428659 --- /dev/null +++ b/backend/common/utils/embedding_lifecycle.py @@ -0,0 +1,112 @@ +"""Missing-vector generation with shared worker locks and conditional publication. + +Content mutations do not take the worker lock: editing remains available while a +provider is slow. Writers must clear embedding atomically with input changes. +Publication compares that input and requires an empty vector, so a stale response +cannot undo invalidation or replace a newer result. A deleted row stays deleted. +""" + +import hashlib + +from sqlalchemy import select, text, update + +from apps.ai_model.embedding import EmbeddingModelCache +from apps.datasource.embedding.utils import parse_valid_embedding +from common.core.embedding_config import load_embedding_config +from common.utils.utils import SQLBotLogUtil + + +class EmbeddingGenerationError(RuntimeError): + """A bounded generation attempt left resources pending for a later retry.""" + + def __init__(self, resource: str, ids: list[int]): + self.resource = resource + self.ids = tuple(ids) + super().__init__(f"Embedding generation failed for {resource}: {self.ids}") + + +def lock_embedding_resource(session, resource: str, resource_id: int) -> None: + """Serialize workers through commit/rollback, without locking business rows. + + All workers (startup, reenable and resource jobs) use the same resource key. + Acquire multiple resources in ascending ID order within one resource type. + PostgreSQL releases transaction locks on rollback, connection loss or death; + there is no process-local success marker that could hide unfinished work. + """ + if session.get_bind().dialect.name == "postgresql": + digest = hashlib.sha256( + f"sqlbot:embedding:{resource}:{resource_id}".encode() + ).digest() + key = int.from_bytes(digest[:8], "big", signed=True) + session.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": key}) + + +def generate_missing_embeddings(session_maker, model, input_column, condition=None): + """Run a knowledge task using its effective config at actual worker start. + + Batch provider calls retain the existing throughput advantage. Lock then + reread avoids generating vectors already completed by another process. A + failed batch gets three attempts; failures remain NULL and are reported to + the Future after other batches have had their chance, never marked complete. + """ + session = session_maker() + failed = [] + try: + if not load_embedding_config(session).enabled: + return + query = select(model.id).where(model.embedding.is_(None)).order_by(model.id) + if condition is not None: + query = query.where(condition) + ids = session.execute(query).scalars().all() + session.commit() + for start in range(0, len(ids), 32): + batch = ids[start : start + 32] + for attempt in range(3): + try: + for resource_id in batch: + lock_embedding_resource( + session, model.__tablename__, resource_id + ) + # Read scalar tuples, not expiring ORM objects. The exact + # original input must survive commits and concurrent edits. + inputs = session.execute( + select(model.id, input_column) + .where(model.id.in_(batch), model.embedding.is_(None)) + .order_by(model.id) + ).all() + if inputs: + vectors = EmbeddingModelCache.get_model().embed_documents( + [value for _, value in inputs] + ) + if len(vectors) != len(inputs): + raise ValueError( + "Embedding provider returned the wrong vector count" + ) + for (resource_id, original_input), vector in zip( + inputs, vectors, strict=True + ): + vector = parse_valid_embedding(vector) + session.execute( + update(model) + .where( + model.id == resource_id, + input_column == original_input, + model.embedding.is_(None), + ) + .values(embedding=vector) + ) + session.commit() + break + except Exception: + session.rollback() + if attempt == 2: + failed.extend(batch) + SQLBotLogUtil.exception( + "Embedding generation remains pending for %s: %s", + model.__tablename__, + batch, + ) + if failed: + raise EmbeddingGenerationError(model.__tablename__, failed) + finally: + session_maker.remove() diff --git a/backend/common/utils/embedding_threads.py b/backend/common/utils/embedding_threads.py index d4a2fe6b..b698f67f 100644 --- a/backend/common/utils/embedding_threads.py +++ b/backend/common/utils/embedding_threads.py @@ -15,34 +15,34 @@ def run_save_terminology_embeddings(ids: List[int]): from apps.terminology.curd.terminology import save_embeddings - executor.submit(save_embeddings, session_maker, ids) + return executor.submit(save_embeddings, session_maker, ids) def fill_empty_terminology_embeddings(): from apps.terminology.curd.terminology import run_fill_empty_embeddings - executor.submit(run_fill_empty_embeddings, session_maker) + return executor.submit(run_fill_empty_embeddings, session_maker) def run_save_data_training_embeddings(ids: List[int]): from apps.data_training.curd.data_training import save_embeddings - executor.submit(save_embeddings, session_maker, ids) + return executor.submit(save_embeddings, session_maker, ids) def fill_empty_data_training_embeddings(): from apps.data_training.curd.data_training import run_fill_empty_embeddings - executor.submit(run_fill_empty_embeddings, session_maker) + return executor.submit(run_fill_empty_embeddings, session_maker) def run_save_table_embeddings(ids: List[int]): from apps.datasource.crud.table import save_table_embedding - executor.submit(save_table_embedding, session_maker, ids) + return executor.submit(save_table_embedding, session_maker, ids) def run_save_ds_embeddings(ids: List[int]): from apps.datasource.crud.table import save_ds_embedding - executor.submit(save_ds_embedding, session_maker, ids) + return executor.submit(save_ds_embedding, session_maker, ids) def fill_empty_table_and_ds_embeddings(): from apps.datasource.crud.table import run_fill_empty_table_and_ds_embedding - executor.submit(run_fill_empty_table_and_ds_embedding, session_maker) + return executor.submit(run_fill_empty_table_and_ds_embedding, session_maker) diff --git a/backend/locales/en.json b/backend/locales/en.json index d40e9d81..20a8b9f7 100644 --- a/backend/locales/en.json +++ b/backend/locales/en.json @@ -205,5 +205,8 @@ "name": "Name", "account": "Account", "email": "Email" + }, + "i18n_parameter": { + "invalid_embedding_parameter": "Invalid embedding parameter: {key}. Similarities and weights must be finite numbers from 0 to 1; candidate counts must be positive integers." } -} \ No newline at end of file +} diff --git a/backend/locales/ko-KR.json b/backend/locales/ko-KR.json index 71e58fa6..14015ea0 100644 --- a/backend/locales/ko-KR.json +++ b/backend/locales/ko-KR.json @@ -205,5 +205,8 @@ "name": "이름", "account": "계정", "email": "이메일" + }, + "i18n_parameter": { + "invalid_embedding_parameter": "벡터 매개변수 {key}이(가) 유효하지 않습니다. 유사도와 가중치는 0~1 사이의 유한한 수, 후보 수는 양의 정수여야 합니다." } -} \ No newline at end of file +} diff --git a/backend/locales/th-TH.json b/backend/locales/th-TH.json index eb2af0d9..4aed4745 100644 --- a/backend/locales/th-TH.json +++ b/backend/locales/th-TH.json @@ -205,5 +205,8 @@ "name": "ชื่อ", "account": "บัญชี", "email": "อีเมล" + }, + "i18n_parameter": { + "invalid_embedding_parameter": "พารามิเตอร์เวกเตอร์ {key} ไม่ถูกต้อง ความคล้ายคลึงและน้ำหนักต้องเป็นจำนวนจำกัดระหว่าง 0 ถึง 1 และจำนวนรายการต้องเป็นจำนวนเต็มบวก" } } diff --git a/backend/locales/zh-CN.json b/backend/locales/zh-CN.json index 7c147b0d..a8c3aa86 100644 --- a/backend/locales/zh-CN.json +++ b/backend/locales/zh-CN.json @@ -205,5 +205,8 @@ "name": "姓名", "account": "账号", "email": "邮箱" + }, + "i18n_parameter": { + "invalid_embedding_parameter": "向量参数 {key} 无效。相似度和权重必须是 0～1 的有限数值，候选数量必须是正整数。" } -} \ No newline at end of file +} diff --git a/backend/locales/zh-TW.json b/backend/locales/zh-TW.json index 24f98a22..1a9d80cd 100644 --- a/backend/locales/zh-TW.json +++ b/backend/locales/zh-TW.json @@ -205,5 +205,8 @@ "name": "姓名", "account": "帳號", "email": "郵箱" + }, + "i18n_parameter": { + "invalid_embedding_parameter": "向量參數 {key} 無效。相似度與權重必須是 0～1 的有限數值，候選數量必須是正整數。" } -} \ No newline at end of file +} diff --git a/backend/main.py b/backend/main.py index 7b187256..740090e2 100644 --- a/backend/main.py +++ b/backend/main.py @@ -42,17 +42,17 @@ def run_migrations(): command.upgrade(alembic_cfg, "head") -@SingleWorkerGuard.once +# Knowledge tasks deduplicate per resource, so every new worker can recover +# pending vectors even while another worker owns the legacy startup lock. def init_terminology_embedding_data(): fill_empty_terminology_embeddings() -@SingleWorkerGuard.once def init_data_training_embedding_data(): fill_empty_data_training_embeddings() -@SingleWorkerGuard.once +# Per-resource database locks deduplicate startup work and recover after worker death. def init_table_and_ds_embedding(): fill_empty_table_and_ds_embeddings() diff --git a/backend/tests/test_datasource_lifecycle.py b/backend/tests/test_datasource_lifecycle.py new file mode 100644 index 00000000..4da7c702 --- /dev/null +++ b/backend/tests/test_datasource_lifecycle.py @@ -0,0 +1,116 @@ +"""Offline lifecycle regressions through parameter and metadata business entries.""" + +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import JSON, Integer, MetaData +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine +from test_embedding_parameters import parameter_app, save, values + +from apps.ai_model.embedding import EmbeddingModelCache +from apps.datasource.crud.datasource import get_ds, updateField +from apps.datasource.crud.table import get_tables_by_ds_id +from apps.datasource.models.datasource import CoreDatasource, CoreField, CoreTable +from common.core.config import settings +from common.utils import embedding_threads + + +@pytest.fixture +def environment(monkeypatch): + database = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + metadata = MetaData() + for model in (CoreDatasource, CoreTable, CoreField): + table = model.__table__.to_metadata(metadata) + table.c.id.type = Integer() + if model is CoreDatasource: + table.c.table_relation.type = JSON() + metadata.create_all(database) + maker = scoped_session(sessionmaker(bind=database, class_=Session)) + jobs = [] + monkeypatch.setattr(embedding_threads, "session_maker", maker) + monkeypatch.setattr( + embedding_threads, + "executor", + SimpleNamespace(submit=lambda function, *args: jobs.append((function, args))), + ) + monkeypatch.setattr(settings, "TABLE_EMBEDDING_ENABLED", True) + with TestClient( + parameter_app(database, monkeypatch), raise_server_exceptions=False + ) as client: + with Session(database) as session: + session.add(CoreDatasource(id=1, name="test", type="pg", embedding="[1,0]")) + session.add(CoreTable(id=1, ds_id=1, table_name="sales", embedding="[1,0]")) + session.add( + CoreField( + id=1, ds_id=1, table_id=1, field_name="id", field_type="INTEGER" + ) + ) + session.commit() + yield database, client, jobs + maker.remove() + database.dispose() + + +def run_jobs(jobs): + while jobs: + function, args = jobs.pop(0) + function(*args) + + +def test_queued_metadata_task_observes_switch_at_actual_start(environment, monkeypatch): + database, client, jobs = environment + with Session(database) as session: + updateField( + session, + CoreField( + id=1, ds_id=1, table_id=1, checked=True, custom_comment="updated" + ), + ) + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": False}]).status_code + == 200 + ) + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: pytest.fail("disabled worker contacted provider"), + ) + run_jobs(jobs) + with Session(database) as session: + assert get_ds(session, 1).embedding is None + assert get_tables_by_ds_id(session, 1)[0].embedding is None + assert settings.TABLE_EMBEDDING_ENABLED is True + assert values(client)["embedding.table_enabled"] == "false" + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_query=lambda _: [0.6, 0.8]), + ) + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": True}]).status_code + == 200 + ) + run_jobs(jobs) + with Session(database) as session: + assert get_ds(session, 1).embedding == "[0.6, 0.8]" + assert get_tables_by_ds_id(session, 1)[0].embedding == "[0.6, 0.8]" + + +@pytest.mark.parametrize("value", [0, 1, "yes", "0", None, [], 0.5]) +def test_invalid_table_switch_rejects_entire_save(environment, value): + _database, client, _jobs = environment + before = values(client) + response = save( + client, + [ + {"pkey": "embedding.table_enabled", "pval": value}, + {"pkey": "embedding.table_count", "pval": 99}, + ], + ) + assert response.status_code >= 400 + assert values(client) == before diff --git a/backend/tests/test_datasource_lifecycle_integration.py b/backend/tests/test_datasource_lifecycle_integration.py new file mode 100644 index 00000000..8203e988 --- /dev/null +++ b/backend/tests/test_datasource_lifecycle_integration.py @@ -0,0 +1,671 @@ +"""Opt-in datasource lifecycle acceptance at parameter/resource/task entries.""" + +from concurrent.futures import ThreadPoolExecutor + +import pytest +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlmodel import Session +from test_embedding_parameters import save, values +from test_embedding_parameters_integration import ( # noqa: F401 + knowledge_environment, + pytestmark, +) +from test_table_parameters_integration import table_environment # noqa: F401 + + +@pytest.fixture +def lifecycle_environment(table_environment, monkeypatch): # noqa: F811 + database, client, ds_id = table_environment + sessions = scoped_session(sessionmaker(bind=database, class_=Session)) + monkeypatch.setattr("common.utils.embedding_threads.session_maker", sessions) + monkeypatch.setattr("apps.system.schemas.permission.engine", database) + with ThreadPoolExecutor(max_workers=4) as executor: + monkeypatch.setattr("common.utils.embedding_threads.executor", executor) + yield database, client, ds_id + sessions.remove() + + +def test_table_switch_is_read_and_saved_through_parameter_api(lifecycle_environment): + _database, client, _ds_id = lifecycle_environment + assert values(client)["embedding.table_enabled"] == "true" + response = save(client, [{"pkey": "embedding.table_enabled", "pval": False}]) + assert response.status_code == 200, response.text + assert values(client)["embedding.table_enabled"] == "false" + + +def test_field_edit_while_disabled_invalidates_table_and_parent( + lifecycle_environment, monkeypatch +): + from sqlmodel import Session + + from apps.datasource.crud.datasource import get_ds, updateField + from apps.datasource.crud.table import get_tables_by_ds_id + from apps.datasource.models.datasource import CoreField + from common.core.config import settings + + database, client, ds_id = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": False}]).status_code + == 200 + ) + monkeypatch.setattr(settings, "TABLE_EMBEDDING_ENABLED", False) + with Session(database) as session: + table = get_tables_by_ds_id(session, ds_id)[0] + table_id = table.id + field = session.query(CoreField).filter(CoreField.table_id == table.id).first() + updateField( + session, + CoreField( + id=field.id, + ds_id=ds_id, + table_id=table.id, + custom_comment="updated field", + checked=True, + ), + ) + with Session(database) as session: + assert get_ds(session, ds_id).embedding is None + assert ( + next( + t for t in get_tables_by_ds_id(session, ds_id) if t.id == table_id + ).embedding + is None + ) + + +def test_generation_reads_persisted_switch_at_task_start( + lifecycle_environment, monkeypatch +): + from types import SimpleNamespace + + from sqlalchemy.orm import scoped_session, sessionmaker + from sqlmodel import Session + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.datasource.crud.datasource import get_ds + from apps.datasource.crud.table import run_fill_empty_table_and_ds_embedding + from apps.datasource.models.datasource import CoreDatasource, CoreTable + + database, client, ds_id = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": False}]).status_code + == 200 + ) + sessions = scoped_session(sessionmaker(bind=database, class_=Session)) + with Session(database) as session: + session.query(CoreDatasource).update({"embedding": None}) + session.query(CoreTable).update({"embedding": None}) + session.commit() + inputs = [] + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace( + embed_query=lambda value: inputs.append(value) or [1, 0] + ), + ) + run_fill_empty_table_and_ds_embedding(sessions) + assert inputs == [] + with Session(database) as session: + assert get_ds(session, ds_id).embedding is None + + +@pytest.mark.parametrize("operation", ["rename", "table", "sync", "choose"]) +def test_metadata_operations_invalidate_parent_while_disabled( + lifecycle_environment, monkeypatch, operation +): + from types import SimpleNamespace + + from apps.datasource.crud import datasource as crud + from apps.datasource.crud.table import get_tables_by_ds_id + from apps.datasource.models.datasource import ( + ColumnSchema, + CoreDatasource, + CoreTable, + ) + + database, client, ds_id = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": False}]).status_code + == 200 + ) + monkeypatch.setattr( + crud, + "getFieldsByDs", + lambda *_args: [ColumnSchema("updated_id", "BIGINT", "synced")], + ) + monkeypatch.setattr(crud, "get_tables", lambda *_args: []) + monkeypatch.setattr(crud, "check_status", lambda *_args: True) + with Session(database) as session: + table = get_tables_by_ds_id(session, ds_id)[0] + table_id = table.id + if operation == "rename": + crud.update_ds( + session, + lambda value: value, + SimpleNamespace(oid=1), + CoreDatasource(id=ds_id, name="renamed", description="new description"), + ) + elif operation == "table": + crud.updateTable( + session, + CoreTable( + id=table_id, ds_id=ds_id, checked=False, custom_comment="new table" + ), + ) + elif operation == "sync": + monkeypatch.setattr( + crud, + "getTablesByDs", + lambda *_args: [SimpleNamespace(tableName=table.table_name)], + ) + crud.sync_single_fields(session, lambda value: value, ds_id, table_id) + else: + crud.chooseTables(session, lambda value: value, ds_id, []) + with Session(database) as session: + assert crud.get_ds(session, ds_id).embedding is None + if operation in ("table", "sync"): + assert ( + next( + t for t in get_tables_by_ds_id(session, ds_id) if t.id == table_id + ).embedding + is None + ) + elif operation == "choose": + assert get_tables_by_ds_id(session, ds_id) == [] + + +def test_backfill_repairs_invalid_json_but_keeps_valid_vectors( + lifecycle_environment, monkeypatch +): + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.datasource.crud.datasource import get_ds + from apps.datasource.crud.table import ( + get_tables_by_ds_id, + run_fill_empty_table_and_ds_embedding, + ) + from apps.datasource.models.datasource import CoreDatasource, CoreTable + + database, _client, ds_id = lifecycle_environment + sessions = scoped_session(sessionmaker(bind=database, class_=Session)) + with Session(database) as session: + tables = get_tables_by_ds_id(session, ds_id) + ids = [table.id for table in tables] + for table, vector in zip( + tables, ["[]", "not-json", "[0,0]", "[0,1]"], strict=True + ): + table.embedding = vector + session.add(table) + session.get(CoreDatasource, ds_id).embedding = "null" + session.commit() + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_query=lambda _value: [1, 0]), + ) + try: + run_fill_empty_table_and_ds_embedding(sessions) + with Session(database) as session: + assert [session.get(CoreTable, i).embedding for i in ids] == [ + "[1.0, 0.0]" + ] * 3 + ["[0,1]"] + assert get_ds(session, ds_id).embedding == "[1.0, 0.0]" + finally: + sessions.remove() + + +def test_imported_schema_invalidates_edited_tables_and_parent(lifecycle_environment): + import io + + import pandas as pd + + from apps.datasource.api.datasource import ( + f_c_col, + f_n_col, + t_c_col, + t_n_col, + t_s_col, + t_sheet, + upload_ds_schema, + ) + from apps.datasource.crud.datasource import get_ds + from apps.datasource.crud.table import get_tables_by_ds_id + + database, client, ds_id = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": False}]).status_code + == 200 + ) + # Route handler and real permission middleware, scoped session and admin identity + # are inherited from the parameter app; only the external upload is synthetic. + client.app.add_api_route("/test/schema/{id}", upload_ds_schema, methods=["POST"]) + with Session(database) as session: + table = get_tables_by_ds_id(session, ds_id)[0] + table_name, table_id = table.table_name, table.id + workbook = io.BytesIO() + with pd.ExcelWriter(workbook, engine="openpyxl") as writer: + pd.DataFrame( + [{t_s_col: "fields", t_n_col: table_name, t_c_col: "Imported table"}] + ).to_excel(writer, sheet_name=t_sheet, index=False) + pd.DataFrame([{f_n_col: "id", f_c_col: "Imported field"}]).to_excel( + writer, sheet_name="fields", index=False + ) + response = client.post( + f"/test/schema/{ds_id}", + files={ + "file": ( + "schema.xlsx", + workbook.getvalue(), + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + }, + ) + assert response.status_code == 200, response.text + with Session(database) as session: + assert get_ds(session, ds_id).embedding is None + edited = next( + table + for table in get_tables_by_ds_id(session, ds_id) + if table.id == table_id + ) + assert edited.custom_comment == "Imported table" + assert edited.embedding is None + assert all( + table.embedding in ("[1,0]", "[0,1]", "[0.8,0.6]") + for table in get_tables_by_ds_id(session, ds_id) + if table.id != table_id + ) + + +def test_reenable_asynchronously_fills_invalidated_metadata( + lifecycle_environment, monkeypatch +): + import time + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.datasource.crud.datasource import get_ds, updateField + from apps.datasource.crud.table import get_tables_by_ds_id + from apps.datasource.models.datasource import CoreField + + database, client, ds_id = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": False}]).status_code + == 200 + ) + with Session(database) as session: + table = get_tables_by_ds_id(session, ds_id)[0] + field = session.query(CoreField).filter(CoreField.table_id == table.id).first() + updateField( + session, + CoreField( + id=field.id, + ds_id=ds_id, + table_id=table.id, + checked=True, + custom_comment="after enable", + ), + ) + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_query=lambda _value: [0.6, 0.8]), + ) + started = time.monotonic() + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": True}]).status_code + == 200 + ) + assert time.monotonic() - started < 1 + while time.monotonic() - started < 3: + with Session(database) as session: + vectors = [t.embedding for t in get_tables_by_ds_id(session, ds_id)] + if get_ds(session, ds_id).embedding == "[0.6, 0.8]" and all(vectors): + break + time.sleep(0.02) + else: + pytest.fail("enabling did not fill missing metadata vectors") + assert vectors.count("[0.6, 0.8]") == 1 + + +@pytest.mark.parametrize("resource", ["table", "datasource", "datasource_field"]) +def test_in_flight_generation_never_publishes_superseded_metadata( + lifecycle_environment, monkeypatch, resource +): + from threading import Event + from types import SimpleNamespace + + from test_table_parameters_integration import configure, run, task + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.datasource.crud import datasource as crud + from apps.datasource.crud.table import ( + get_tables_by_ds_id, + save_ds_embedding, + save_table_embedding, + ) + from apps.datasource.models.datasource import CoreDatasource, CoreField + + database, client, ds_id = lifecycle_environment + sessions = scoped_session(sessionmaker(bind=database, class_=Session)) + entered, release = Event(), Event() + inputs = [] + + def embed(value): + inputs.append(value) + if len(inputs) == 1: + entered.set() + assert release.wait(5) + return [0, 1] if "latest content" in value else [1, 0] + + monkeypatch.setattr( + EmbeddingModelCache, "get_model", lambda: SimpleNamespace(embed_query=embed) + ) + with Session(database) as session: + table = next(t for t in get_tables_by_ds_id(session, ds_id) if t.checked) + table_id = table.id + field = session.query(CoreField).filter(CoreField.table_id == table_id).first() + field_id = field.id + if resource == "table": + table.embedding = None + else: + session.get(CoreDatasource, ds_id).embedding = None + session.commit() + generator = save_table_embedding if resource == "table" else save_ds_embedding + with ThreadPoolExecutor(max_workers=1) as workers: + future = workers.submit( + generator, sessions, [table_id if resource == "table" else ds_id] + ) + try: + assert entered.wait(5) + # Stop new jobs, while allowing the already-started one to finish. + configure(client, table_enabled=False) + with Session(database) as session: + if resource in ("table", "datasource_field"): + crud.updateField( + session, + CoreField( + id=field_id, + ds_id=ds_id, + table_id=table_id, + checked=True, + custom_comment="latest content", + ), + ) + else: + crud.update_ds( + session, + lambda value: value, + SimpleNamespace(oid=1), + CoreDatasource( + id=ds_id, + name="latest content", + description="latest content", + ), + ) + with Session(database) as session: + assert crud.get_ds(session, ds_id).embedding is None + schema = run(task(database, ds_id)) + assert ( + "sales_alpha" in schema + and "vector_beta" in schema + and "other_gamma" in schema + ) + finally: + release.set() + future.result(timeout=5) + with Session(database) as session: + result = ( + next(t for t in get_tables_by_ds_id(session, ds_id) if t.id == table_id) + if resource == "table" + else crud.get_ds(session, ds_id) + ) + assert result.embedding == "[0.0, 1.0]" + if resource == "table": + assert crud.get_ds(session, ds_id).embedding is None + assert "latest content" in inputs[-1] + + +@pytest.mark.parametrize("bad_vector", [[], [0, 0], [float("nan"), 1], ["1", 0], None]) +def test_failed_generation_leaves_candidates_available( + lifecycle_environment, monkeypatch, bad_vector +): + from types import SimpleNamespace + + from test_table_parameters_integration import configure, run, task + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.datasource.crud.table import ( + get_tables_by_ds_id, + run_fill_empty_table_and_ds_embedding, + ) + from apps.datasource.models.datasource import CoreDatasource + from common.utils.embedding_lifecycle import EmbeddingGenerationError + + database, client, ds_id = lifecycle_environment + configure(client, table_count=1) + with Session(database) as session: + table = next(t for t in get_tables_by_ds_id(session, ds_id) if t.checked) + table.embedding = None + session.get(CoreDatasource, ds_id).embedding = None + session.commit() + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_query=lambda _: bad_vector), + ) + sessions = scoped_session(sessionmaker(bind=database, class_=Session)) + with pytest.raises(EmbeddingGenerationError): + run_fill_empty_table_and_ds_embedding(sessions) + schema = run(task(database, ds_id)) + assert all(name in schema for name in ("sales_alpha", "vector_beta", "other_gamma")) + with Session(database) as session: + assert session.get(CoreDatasource, ds_id).embedding is None + assert any( + t.embedding is None + for t in get_tables_by_ds_id(session, ds_id) + if t.checked + ) + + +@pytest.mark.parametrize("crash_first", [False, True]) +def test_process_workers_deduplicate_and_recover_after_crash( + lifecycle_environment, tmp_path, crash_first +): + import json + import os + import subprocess + import sys + import time + from pathlib import Path + + from sqlalchemy import text + + from apps.datasource.crud.table import get_tables_by_ds_id + from apps.datasource.models.datasource import CoreDatasource + + database, _client, ds_id = lifecycle_environment + with Session(database) as session: + table = get_tables_by_ds_id(session, ds_id)[0] + table.embedding = None + session.get(CoreDatasource, ds_id).embedding = None + session.commit() + with database.connect() as connection: + schema = connection.execute(text("select current_schema()")).scalar_one() + application = "fill_" + schema + worker = tmp_path / "worker.py" + worker.write_text(""" +import json, sys, time +from pathlib import Path +from types import SimpleNamespace +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlmodel import Session, create_engine +from apps.ai_model.embedding import EmbeddingModelCache +from common.core.config import settings +from common.utils import embedding_threads +url, schema, app, directory = sys.argv[1:] +directory = Path(directory) +engine = create_engine(url, connect_args={'options': '-csearch_path=' + schema + ',public', 'application_name': app}) +embedding_threads.session_maker = scoped_session(sessionmaker(bind=engine, class_=Session)) +settings.TABLE_EMBEDDING_ENABLED = True +def embed(value): + with (directory / 'generated.jsonl').open('a') as log: + log.write(json.dumps(value) + '\\n') + deadline = time.monotonic() + 25 + while not (directory / 'release').exists(): + if time.monotonic() > deadline: + raise TimeoutError('provider barrier') + time.sleep(0.01) + return [0.6, 0.8] +EmbeddingModelCache.get_model = staticmethod(lambda: SimpleNamespace(embed_query=embed)) +embedding_threads.fill_empty_table_and_ds_embeddings().result(timeout=30) +embedding_threads.executor.shutdown(wait=True) +engine.dispose() +print('completed') +""") + command = [ + sys.executable, + str(worker), + database.url.render_as_string(hide_password=False), + schema, + application, + str(tmp_path), + ] + env = {**os.environ, "PYTHONPATH": str(Path(__file__).parents[1].resolve())} + processes = [] + + def await_state(predicate): + deadline = time.monotonic() + 20 + while not predicate(): + assert time.monotonic() < deadline, "worker progress timed out" + for process in processes: + assert process.poll() is None, process.communicate()[1] + time.sleep(0.02) + + def waiting(): + with database.connect() as connection: + return ( + connection.execute( + text( + "SELECT count(*) FROM pg_stat_activity WHERE application_name=:name AND wait_event='advisory'" + ), + {"name": application}, + ).scalar_one() + == 1 + ) + + try: + first = subprocess.Popen( + command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + processes.append(first) + await_state(lambda: (tmp_path / "generated.jsonl").exists()) + second = subprocess.Popen( + command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + processes.append(second) + await_state(waiting) + if crash_first: + first.kill() + first.communicate(timeout=10) + processes.remove(first) + await_state( + lambda: ( + len((tmp_path / "generated.jsonl").read_text().splitlines()) == 2 + ) + ) + (tmp_path / "release").touch() + for process in processes: + stdout, stderr = process.communicate(timeout=30) + assert process.returncode == 0, stderr + assert "completed" in stdout + finally: + (tmp_path / "release").touch() + for process in processes: + if process.poll() is None: + process.kill() + process.communicate(timeout=10) + generated = [ + json.loads(line) + for line in (tmp_path / "generated.jsonl").read_text().splitlines() + ] + assert len(generated) == (3 if crash_first else 2) + with Session(database) as session: + assert session.get(CoreDatasource, ds_id).embedding == "[0.6, 0.8]" + assert get_tables_by_ds_id(session, ds_id)[0].embedding == "[0.6, 0.8]" + + +def test_create_and_empty_field_sync_recover_after_enable( + lifecycle_environment, monkeypatch +): + import asyncio + import time + from types import SimpleNamespace + + from apps.datasource.crud import datasource as crud + from apps.datasource.crud.table import get_tables_by_ds_id + from apps.datasource.models.datasource import ( + ColumnSchema, + CoreField, + CoreTable, + CreateDatasource, + ) + + database, client, _ds_id = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": False}]).status_code + == 200 + ) + monkeypatch.setattr( + crud, + "getFieldsByDs", + lambda *_args: [ColumnSchema("id", "INTEGER", "created field")], + ) + monkeypatch.setattr( + crud, "get_tables", lambda *_args: [SimpleNamespace(tableName="created")] + ) + monkeypatch.setattr( + crud, "getTablesByDs", lambda *_args: [SimpleNamespace(tableName="created")] + ) + with Session(database) as session: + datasource = asyncio.run( + crud.create_ds( + session, + lambda value: value, + SimpleNamespace(id=1, oid=1), + CreateDatasource( + name="created datasource", + type="pg", + tables=[ + CoreTable(table_name="created", table_comment="created table") + ], + ), + ) + ) + ds_id = datasource.id + table_id = get_tables_by_ds_id(session, ds_id)[0].id + with Session(database) as session: + assert crud.get_ds(session, ds_id).embedding is None + assert get_tables_by_ds_id(session, ds_id)[0].embedding is None + monkeypatch.setattr(crud, "getFieldsByDs", lambda *_args: []) + crud.sync_single_fields(session, lambda value: value, ds_id, table_id) + assert ( + session.query(CoreField).filter(CoreField.table_id == table_id).all() == [] + ) + assert ( + save(client, [{"pkey": "embedding.table_enabled", "pval": True}]).status_code + == 200 + ) + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + with Session(database) as session: + if ( + crud.get_ds(session, ds_id).embedding + and get_tables_by_ds_id(session, ds_id)[0].embedding + ): + break + time.sleep(0.02) + else: + pytest.fail("created datasource remained unfilled") diff --git a/backend/tests/test_embedding_parameters.py b/backend/tests/test_embedding_parameters.py new file mode 100644 index 00000000..4497691c --- /dev/null +++ b/backend/tests/test_embedding_parameters.py @@ -0,0 +1,231 @@ +"""Offline parameter API regressions using the released xpack persistence adapter.""" + +import json +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from fastapi.responses import JSONResponse +from fastapi.testclient import TestClient +from sqlalchemy.pool import StaticPool +from sqlbot_xpack.config.model import SysArgModel +from sqlmodel import Session, create_engine + +from apps.system.api.parameter import router +from apps.system.schemas.permission import RequestContextMiddleware +from common.audit.models.log_model import SystemLog, SystemLogsResource +from common.audit.schemas.request_context import RequestContextMiddlewareCommon +from common.core.config import settings +from common.core.db import get_session + + +def parameter_app(database, monkeypatch): + """Use real auth decorators/auditing; only identity and database are test adapters.""" + for model in (SysArgModel, SystemLog, SystemLogsResource): + model.__table__.create(database, checkfirst=True) + monkeypatch.setattr("common.audit.schemas.logger_decorator.engine", database) + app = FastAPI() + app.include_router(router) + app.add_middleware(RequestContextMiddleware) + app.add_middleware(RequestContextMiddlewareCommon) + + @app.middleware("http") + async def identity(request, call_next): + role = request.headers.get("test-role", "admin") + if role != "anonymous": + request.state.current_user = SimpleNamespace( + isAdmin=role == "admin", + weight=1 if role == "workspace-admin" else 0, + id=1, + oid=1, + username="test", + account="test", + name="test", + ) + return await call_next(request) + + @app.exception_handler(Exception) + async def error_response(_request, _exc): + return JSONResponse(status_code=400, content={"error": str(_exc)}) + + def session(): + with Session(database) as value: + yield value + + app.dependency_overrides[get_session] = session + return app + + +@pytest.fixture +def parameter_client(monkeypatch): + database = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + with TestClient( + parameter_app(database, monkeypatch), raise_server_exceptions=False + ) as client: + yield client + database.dispose() + + +def values(client): + response = client.get("/system/parameter") + assert response.status_code == 200 + return {item["pkey"]: item["pval"] for item in response.json()} + + +def save(client, items, **kwargs): + return client.post("/system/parameter", data={"data": json.dumps(items)}, **kwargs) + + +def test_deployment_defaults_and_saved_knowledge_parameters( + parameter_client, monkeypatch +): + monkeypatch.setattr(settings, "EMBEDDING_TERMINOLOGY_SIMILARITY", 0.65) + assert values(parameter_client)["embedding.terminology_similarity"] == "0.65" + response = save( + parameter_client, + [ + {"pkey": "embedding.terminology_similarity", "pval": 0}, + {"pkey": "embedding.terminology_top_count", "pval": 1}, + {"pkey": "embedding.data_training_similarity", "pval": 1}, + {"pkey": "embedding.data_training_top_count", "pval": 9}, + ], + ) + assert response.status_code == 200 + result = values(parameter_client) + assert result["embedding.terminology_similarity"] == "0.0" + assert result["embedding.terminology_top_count"] == "1" + assert result["embedding.data_training_similarity"] == "1.0" + assert result["embedding.data_training_top_count"] == "9" + + +@pytest.mark.parametrize( + "key,bad_value", + [ + ("terminology_similarity", -0.1), + ("terminology_similarity", 1.1), + ("data_training_similarity", "nan"), + ("data_training_similarity", "inf"), + ("terminology_top_count", True), + ("terminology_top_count", 0), + ("data_training_top_count", 1.5), + ("data_training_top_count", "1.0"), + ("terminology_similarity", None), + ("table_count", 0), + ("table_count", 1.5), + ("datasource_count", -1), + ("datasource_count", True), + ("table_alpha", -0.1), + ("table_alpha", 1.1), + ("table_alpha", "nan"), + ("table_alpha", "inf"), + ("keyword_enabled", 1), + ("keyword_enabled", "yes"), + ], +) +def test_invalid_batch_preserves_all_existing_parameters( + parameter_client, key, bad_value +): + assert ( + save( + parameter_client, [{"pkey": "chat.sqlbot_name", "pval": "existing"}] + ).status_code + == 200 + ) + before = values(parameter_client) + response = save( + parameter_client, + [ + {"pkey": "chat.sqlbot_name", "pval": "changed"}, + {"pkey": "embedding.terminology_top_count", "pval": 8}, + {"pkey": f"embedding.{key}", "pval": bad_value}, + ], + ) + assert response.status_code >= 400 + assert values(parameter_client) == before + + +@pytest.mark.parametrize("role", ["anonymous", "user", "workspace-admin"]) +def test_only_system_admin_can_manage_parameters(parameter_client, role): + before = values(parameter_client) + assert ( + parameter_client.get( + "/system/parameter", headers={"test-role": role} + ).status_code + >= 400 + ) + assert ( + save( + parameter_client, + [{"pkey": "embedding.terminology_top_count", "pval": 12}], + headers={"test-role": role}, + ).status_code + >= 400 + ) + assert values(parameter_client) == before + + +@pytest.mark.parametrize( + "payload", + [ + {"pkey": "embedding.terminology_top_count", "pval": 8}, + [None], + [42], + [{"pkey": 1, "pval": 3}], + ], +) +def test_malformed_batch_is_rejected_without_updates(parameter_client, payload): + before = values(parameter_client) + assert save(parameter_client, payload).status_code >= 400 + assert values(parameter_client) == before + + +@pytest.mark.parametrize("data", [None, "{", "null"]) +def test_malformed_form_has_a_localized_validation_error(parameter_client, data): + before = values(parameter_client) + response = parameter_client.post( + "/system/parameter", data={} if data is None else {"data": data} + ) + assert response.status_code >= 400 + assert response.json()["error"].startswith("Invalid embedding parameter:") + assert values(parameter_client) == before + + +def test_table_selection_parameters_preserve_types_and_other_settings( + parameter_client, monkeypatch +): + monkeypatch.setattr(settings, "TABLE_EMBEDDING_COUNT", 17) + assert values(parameter_client)["embedding.table_count"] == "17" + assert ( + save( + parameter_client, [{"pkey": "chat.sqlbot_name", "pval": "existing"}] + ).status_code + == 200 + ) + for alpha, enabled in [(0, True), (1, "false")]: + assert ( + save( + parameter_client, + [ + {"pkey": "embedding.table_count", "pval": 1}, + {"pkey": "embedding.datasource_count", "pval": 3}, + {"pkey": "embedding.table_alpha", "pval": alpha}, + {"pkey": "embedding.keyword_enabled", "pval": enabled}, + ], + ).status_code + == 200 + ) + result = values(parameter_client) + assert result["embedding.table_count"] == "1" + assert result["embedding.datasource_count"] == "3" + assert result["embedding.table_alpha"] == f"{alpha}.0" + assert result["embedding.keyword_enabled"] == str(enabled).lower() + assert result["chat.sqlbot_name"] == "existing" + entries = parameter_client.get("/system/parameter").json() + assert ( + next(item for item in entries if item["pkey"] == "embedding.keyword_enabled")[ + "ptype" + ] + == "bool" + ) diff --git a/backend/tests/test_embedding_parameters_integration.py b/backend/tests/test_embedding_parameters_integration.py new file mode 100644 index 00000000..e891c118 --- /dev/null +++ b/backend/tests/test_embedding_parameters_integration.py @@ -0,0 +1,431 @@ +"""Opt-in PG/pgvector contract tests. + +Run explicitly with SQLBOT_RUN_EMBEDDING_INTEGRATION=1 and +SQLBOT_TEST_DATABASE_URL pointing at an authorized disposable PostgreSQL database. +Each test owns a fresh schema; no existing application rows are used. +""" + +import asyncio +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest +from fastapi.testclient import TestClient +from langchain_core.messages import AIMessageChunk +from sqlalchemy import text, update +from sqlbot_xpack.config.model import SysArgModel +from sqlmodel import Session, create_engine +from test_embedding_parameters import parameter_app, save, values + +from apps.ai_model.embedding import EmbeddingModelCache +from apps.ai_model.model_factory import LLMConfig +from apps.chat.curd.chat import get_chat_with_records +from apps.chat.models.chat_model import Chat, ChatLog, ChatQuestion, ChatRecord +from apps.chat.task.llm import LLMService +from apps.data_training.models.data_training_model import DataTraining +from apps.system.models.system_model import WorkspaceModel +from apps.terminology.models.terminology_model import Terminology +from common.audit.models.log_model import SystemLog, SystemLogsResource +from common.core.config import settings + +pytestmark = pytest.mark.skipif( + os.environ.get("SQLBOT_RUN_EMBEDDING_INTEGRATION") != "1", + reason="explicit disposable PostgreSQL integration opt-in required", +) + + +@pytest.fixture +def knowledge_environment(monkeypatch, request): + url = os.environ["SQLBOT_TEST_DATABASE_URL"] + schema = "embedding_1404_" + uuid.uuid4().hex + admin = create_engine(url) + with admin.begin() as connection: + connection.execute(text(f'CREATE SCHEMA "{schema}"')) + database = create_engine( + url, connect_args={"options": f"-csearch_path={schema},public"} + ) + + def cleanup(): + database.dispose() + with admin.begin() as connection: + connection.execute(text(f'DROP SCHEMA "{schema}" CASCADE')) + admin.dispose() + + request.addfinalizer(cleanup) + # checkfirst=True sees public tables through search_path; create explicitly in + # our fresh current schema before handing the engine to the real router. + for model in ( + WorkspaceModel, + Chat, + ChatRecord, + ChatLog, + Terminology, + DataTraining, + SystemLog, + SystemLogsResource, + ): + model.__table__.create(database) + # Match the installed migration, which lacks the wheel model's unique pkey. + with database.begin() as connection: + connection.execute( + text( + "CREATE TABLE sys_arg (id BIGINT PRIMARY KEY, pkey VARCHAR(255), pval VARCHAR(255), ptype VARCHAR(255), sort_no INTEGER NOT NULL DEFAULT 0)" + ) + ) + # The audit migration allows NULL operation detail and optional fields; + # its legacy SQLModel annotations are stricter than the real schema. + for column in SystemLog.__table__.columns: + if not column.primary_key and not column.nullable: + connection.execute( + text( + f'ALTER TABLE sys_logs ALTER COLUMN "{column.name}" DROP NOT NULL' + ) + ) + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_query=lambda _query: [1.0, 0.0]), + ) + monkeypatch.setattr(settings, "EMBEDDING_ENABLED", True) + monkeypatch.setattr(settings, "EMBEDDING_TERMINOLOGY_TOP_COUNT", 1) + monkeypatch.setattr(settings, "EMBEDDING_DATA_TRAINING_TOP_COUNT", 1) + monkeypatch.setattr(settings, "EMBEDDING_TERMINOLOGY_SIMILARITY", 0.5) + monkeypatch.setattr(settings, "EMBEDDING_DATA_TRAINING_SIMILARITY", 0.5) + with TestClient( + parameter_app(database, monkeypatch), raise_server_exceptions=False + ) as client: + with Session(database) as session: + session.add(WorkspaceModel(id=1, name="test")) + chat = Chat( + oid=1, + brief="test", + create_by=1, + create_time=datetime.now(), + engine_type="PostgreSQL", + ) + session.add(chat) + session.flush() + chat_id = chat.id + for oid, datasource, application in [ + (1, 10, None), + (1, None, 99), + (2, 10, None), + ]: + for word, vector in [ + ("alpha", [1, 0]), + ("beta", [0.8, 0.6]), + ("literal", [0, 1]), + ]: + session.add( + Terminology( + oid=oid, + word=word, + description=f"term-{word}-{oid}-{application}", + embedding=vector, + enabled=True, + specific_ds=True, + datasource_ids=[datasource] if datasource else [], + advanced_application=application, + ) + ) + session.add( + DataTraining( + oid=oid, + question=word, + description=f"example-{word}-{oid}-{application}", + embedding=vector, + enabled=True, + datasource=datasource, + advanced_application=application, + ) + ) + session.commit() + yield database, client, chat_id + + +def question_task(session, chat_id, assistant=None): + user = SimpleNamespace(id=1, oid=1, language="en") + service = LLMService( + session, + user, + ChatQuestion(chat_id=chat_id, question="literal query"), + current_assistant=assistant, + config=LLMConfig( + model_type="openai", + model_name="test", + api_key="test", + api_base_url="http://127.0.0.1:9/v1", + ), + ) + service.init_record(session) + return service + + +def recall(service, session): + service.filter_terminology_template(session, 1, 10) + service.filter_training_template(session, 1, 10) + return service.chat_question.terminologies, service.chat_question.data_training + + +def test_new_question_uses_saved_counts_and_in_flight_question_keeps_snapshot( + knowledge_environment, +): + database, client, chat_id = knowledge_environment + with Session(database) as session: + old_task = question_task(session, chat_id) + before = recall(old_task, session) + assert "term-alpha-1-None" in before[0] and "term-beta" not in before[0] + assert "example-alpha-1-None" in before[1] and "example-beta" not in before[1] + assert ( + save( + client, + [ + {"pkey": "embedding.terminology_top_count", "pval": 2}, + {"pkey": "embedding.data_training_top_count", "pval": 2}, + ], + ).status_code + == 200 + ) + assert recall(old_task, session) == before + with Session(database) as session: + result = recall(question_task(session, chat_id), session) + assert "term-beta-1-None" in result[0] + assert "example-beta-1-None" in result[1] + assert ( + "term-literal-1-None" in result[0] and "example-literal-1-None" in result[1] + ) + assert "-2-" not in "".join(result) + assert "-1-99" not in "".join(result) + + +def test_strict_threshold_preserves_literal_matches_and_advanced_knowledge_scope( + knowledge_environment, +): + database, client, chat_id = knowledge_environment + assert ( + save( + client, + [ + {"pkey": "embedding.terminology_similarity", "pval": 1}, + {"pkey": "embedding.data_training_similarity", "pval": 1}, + ], + ).status_code + == 200 + ) + with Session(database) as session: + result = recall(question_task(session, chat_id), session) + assert "alpha" not in "".join(result) + assert "literal" in result[0] and "literal" in result[1] + advanced = question_task( + session, chat_id, SimpleNamespace(type=1, id=99, oid=1) + ) + result = recall(advanced, session) + assert "term-literal-1-99" in result[0] and "example-literal-1-99" in result[1] + assert "-1-None" not in "".join(result) + assert values(client)["embedding.terminology_similarity"] == "1.0" + + +def test_analysis_retrieves_terms_using_new_settings_and_historical_keywords( + knowledge_environment, +): + database, client, chat_id = knowledge_environment + with Session(database) as session: + session.add( + Terminology( + oid=1, + word="literal", + description="historical-keyword", + embedding=[0, 1], + specific_ds=False, + ) + ) + session.add( + Terminology( + oid=1, + word="global-vector", + description="vector-only", + embedding=[1, 0], + specific_ds=False, + ) + ) + session.commit() + old_task = question_task(session, chat_id) + assert ( + save( + client, + [ + {"pkey": "embedding.terminology_similarity", "pval": 1}, + {"pkey": "embedding.keyword_enabled", "pval": False}, + ], + ).status_code + == 200 + ) + new_task = question_task(session, chat_id) + messages = [] + + def stream(value): + messages.extend(value) + return iter([AIMessageChunk(content="analysis result")]) + + for task in (old_task, new_task): + task.record.extracted_keywords = "literal" + task.chat_question.question = "unrelated original question" + session.execute( + update(ChatRecord) + .where(ChatRecord.id == task.record.id) + .values(extracted_keywords="literal") + ) + session.commit() + task.llm = SimpleNamespace(stream=stream) + list(task.generate_analysis(session)) + assert "vector-only" in old_task.chat_question.terminologies + assert "vector-only" not in new_task.chat_question.terminologies + assert "historical-keyword" in new_task.chat_question.terminologies + assert any("historical-keyword" in str(message.content) for message in messages) + records = get_chat_with_records( + session, chat_id, new_task.current_user, None + ).records + assert any(record["analysis"] == "analysis result" for record in records) + + +def test_new_task_in_another_process_uses_api_saved_settings(knowledge_environment): + database, client, chat_id = knowledge_environment + assert ( + save( + client, + [ + {"pkey": "embedding.terminology_top_count", "pval": 1}, + {"pkey": "embedding.data_training_top_count", "pval": 1}, + ], + ).status_code + == 200 + ) + with database.connect() as connection: + schema = connection.execute(text("select current_schema()")).scalar_one() + script = """ +import json, os +from types import SimpleNamespace +from sqlmodel import Session, create_engine +from apps.ai_model.embedding import EmbeddingModelCache +from test_embedding_parameters_integration import question_task, recall +EmbeddingModelCache.get_model = staticmethod(lambda: SimpleNamespace(embed_query=lambda _: [1.0, 0.0])) +db = create_engine(os.environ['SQLBOT_TEST_DATABASE_URL'], connect_args={'options': '-csearch_path=' + os.environ['SQLBOT_TEST_SCHEMA'] + ',public'}) +with Session(db) as session: + result = recall(question_task(session, int(os.environ['SQLBOT_TEST_CHAT_ID'])), session) + print('RESULT:' + json.dumps(result)) +db.dispose() +""" + env = { + **os.environ, + "EMBEDDING_ENABLED": "true", + "EMBEDDING_TERMINOLOGY_SIMILARITY": "0.5", + "EMBEDDING_DATA_TRAINING_SIMILARITY": "0.5", + "EMBEDDING_TERMINOLOGY_TOP_COUNT": "5", + "EMBEDDING_DATA_TRAINING_TOP_COUNT": "5", + "SQLBOT_TEST_SCHEMA": schema, + "SQLBOT_TEST_CHAT_ID": str(chat_id), + "PYTHONPATH": os.pathsep.join( + [ + str(Path(__file__).parent.resolve()), + str(Path(__file__).parents[1].resolve()), + ] + ), + } + process = subprocess.run( + [sys.executable, "-c", script], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert process.returncode == 0, process.stderr + result = json.loads( + next( + line.removeprefix("RESULT:") + for line in process.stdout.splitlines() + if line.startswith("RESULT:") + ) + ) + # The child process has startup count=5, so beta's exclusion proves it read + # the persisted count rather than merely matching the startup defaults. + assert "term-alpha-1-None" in result[0] and "term-beta" not in result[0] + assert "example-alpha-1-None" in result[1] and "example-beta" not in result[1] + + +def test_legacy_duplicate_keys_have_deterministic_effective_value_and_can_be_saved( + knowledge_environment, +): + database, client, _chat_id = knowledge_environment + with Session(database) as session: + session.add( + SysArgModel(id=100, pkey="embedding.terminology_top_count", pval="3") + ) + session.add( + SysArgModel(id=101, pkey="embedding.terminology_top_count", pval="7") + ) + session.commit() + assert values(client)["embedding.terminology_top_count"] == "7" + assert ( + save( + client, [{"pkey": "embedding.terminology_top_count", "pval": 4}] + ).status_code + == 200 + ) + assert values(client)["embedding.terminology_top_count"] == "4" + + +def test_waiting_for_another_parameter_writer_does_not_block_requests( + knowledge_environment, monkeypatch +): + database, _client, _chat_id = knowledge_environment + app = parameter_app(database, monkeypatch) + ready = threading.Event() + + def hold_lock(): + with database.begin() as connection: + connection.execute( + text("SELECT pg_advisory_xact_lock(:key)"), {"key": 14030001} + ) + ready.set() + time.sleep(1) + + holder = threading.Thread(target=hold_lock) + holder.start() + assert ready.wait(timeout=2) + + async def requests(): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app), base_url="http://testserver" + ) as client: + started = time.monotonic() + writer = asyncio.create_task( + client.post( + "/system/parameter", + data={ + "data": json.dumps( + [ + {"pkey": "embedding.terminology_top_count", "pval": 4}, + ] + ) + }, + ) + ) + await asyncio.sleep(0.1) + response = await client.get("/system/parameter") + assert response.status_code == 200 + assert time.monotonic() - started < 0.5 + assert (await writer).status_code == 200 + + try: + asyncio.run(requests()) + finally: + holder.join(timeout=3) diff --git a/backend/tests/test_knowledge_lifecycle.py b/backend/tests/test_knowledge_lifecycle.py new file mode 100644 index 00000000..e9f7af58 --- /dev/null +++ b/backend/tests/test_knowledge_lifecycle.py @@ -0,0 +1,188 @@ +"""Offline API and resource-task regressions; providers never use the network.""" + +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import Integer, MetaData +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine +from test_embedding_parameters import parameter_app, save, values + +from apps.ai_model.embedding import EmbeddingModelCache +from apps.data_training.curd.data_training import create_training, update_training +from apps.data_training.models.data_training_model import DataTraining, DataTrainingInfo +from common.core.config import settings +from common.utils import embedding_threads +from common.utils.embedding_lifecycle import EmbeddingGenerationError + + +@pytest.fixture +def environment(monkeypatch): + database = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + # SQLite's auto-ID spelling differs; the resource/API/task code stays real. + metadata = MetaData() + table = DataTraining.__table__.to_metadata(metadata) + table.c.id.type = Integer() + metadata.create_all(database) + maker = scoped_session(sessionmaker(bind=database)) + jobs = [] + monkeypatch.setattr(embedding_threads, "session_maker", maker) + monkeypatch.setattr( + embedding_threads, + "executor", + SimpleNamespace(submit=lambda function, *args: jobs.append((function, args))), + ) + monkeypatch.setattr(settings, "EMBEDDING_ENABLED", True) + with TestClient( + parameter_app(database, monkeypatch), raise_server_exceptions=False + ) as client: + yield database, client, jobs + maker.remove() + database.dispose() + + +def run_jobs(jobs): + while jobs: + function, args = jobs.pop(0) + function(*args) + + +def add_example(database, question="queued question"): + with Session(database) as session: + return create_training( + session, + DataTrainingInfo(question=question, description="select 1", datasource=10), + 1, + lambda key: key, + ) + + +def test_queued_resource_task_uses_persisted_switch_when_it_starts( + environment, monkeypatch +): + database, client, jobs = environment + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: pytest.fail("disabled task contacted provider"), + ) + resource_id = add_example(database) + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code == 200 + ) + run_jobs(jobs) + with Session(database) as session: + assert session.get(DataTraining, resource_id).embedding is None + assert settings.EMBEDDING_ENABLED is True + assert values(client)["embedding.enabled"] == "false" + + +@pytest.mark.parametrize("value", [0, 1, "yes", "0", None, [], 0.5]) +def test_invalid_switch_rejects_whole_parameter_save(environment, value): + _database, client, _jobs = environment + before = values(client) + assert ( + save( + client, + [ + {"pkey": "embedding.enabled", "pval": value}, + {"pkey": "embedding.terminology_top_count", "pval": 99}, + ], + ).status_code + >= 400 + ) + assert values(client) == before + + +def test_provider_failure_is_visible_and_pending_resource_can_retry( + environment, monkeypatch +): + database, _client, jobs = environment + resource_id = add_example(database) + + def fail(_words): + raise ConnectionError("controlled provider unavailable") + + monkeypatch.setattr( + EmbeddingModelCache, "get_model", lambda: SimpleNamespace(embed_documents=fail) + ) + with pytest.raises(EmbeddingGenerationError): + run_jobs(jobs) + with Session(database) as session: + assert session.get(DataTraining, resource_id).embedding is None + update_training( + session, + DataTrainingInfo( + id=resource_id, + question="queued question", + description="select 2", + datasource=10, + ), + 1, + lambda key: key, + ) + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_documents=lambda words: [[1, 0] for _ in words]), + ) + run_jobs(jobs) + with Session(database) as session: + assert list(session.get(DataTraining, resource_id).embedding) == [1, 0] + + +def test_transient_provider_failure_recovers_within_task(environment, monkeypatch): + database, _client, jobs = environment + resource_id = add_example(database) + responses = iter([ConnectionError("temporary"), [[0.6, 0.8]]]) + + def embed(_words): + result = next(responses) + if isinstance(result, Exception): + raise result + return result + + monkeypatch.setattr( + EmbeddingModelCache, "get_model", lambda: SimpleNamespace(embed_documents=embed) + ) + run_jobs(jobs) + with Session(database) as session: + assert list(session.get(DataTraining, resource_id).embedding) == pytest.approx( + [0.6, 0.8] + ) + + +@pytest.mark.parametrize("role", ["anonymous", "user", "workspace-admin"]) +def test_only_system_admin_can_toggle_knowledge(environment, role): + _database, client, _jobs = environment + before = values(client) + assert ( + save( + client, + [{"pkey": "embedding.enabled", "pval": False}], + headers={"test-role": role}, + ).status_code + >= 400 + ) + assert values(client) == before + + +@pytest.mark.parametrize( + "vector", [[0, 0], [float("nan"), 1], [float("inf"), 1], [True, 0], ["1", 0], []] +) +def test_unusable_provider_vectors_remain_pending(environment, monkeypatch, vector): + database, _client, jobs = environment + resource_id = add_example(database) + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_documents=lambda _words: [vector]), + ) + with pytest.raises(EmbeddingGenerationError): + run_jobs(jobs) + with Session(database) as session: + assert session.get(DataTraining, resource_id).embedding is None diff --git a/backend/tests/test_knowledge_lifecycle_integration.py b/backend/tests/test_knowledge_lifecycle_integration.py new file mode 100644 index 00000000..d5756b53 --- /dev/null +++ b/backend/tests/test_knowledge_lifecycle_integration.py @@ -0,0 +1,939 @@ +"""Opt-in knowledge resource/task regressions on disposable PG/pgvector.""" + +import os +from concurrent.futures import Future + +import pytest +import test_embedding_parameters_integration as parameter_contracts +from sqlmodel import Session, select +from test_embedding_parameters import save, values +from test_embedding_parameters_integration import ( + question_task, + recall, +) + +from apps.data_training.curd.data_training import update_training +from apps.data_training.models.data_training_model import DataTraining, DataTrainingInfo +from apps.terminology.curd.terminology import update_terminology +from apps.terminology.models.terminology_model import Terminology, TerminologyInfo +from common.utils import embedding_threads + +knowledge_environment = parameter_contracts.knowledge_environment + +pytestmark = pytest.mark.skipif( + os.environ.get("SQLBOT_RUN_EMBEDDING_INTEGRATION") != "1", + reason="explicit disposable PostgreSQL integration opt-in required", +) + + +class QueuedExecutor: + """Controllable scheduling boundary; jobs run only when the test starts them.""" + + def __init__(self): + self.jobs = [] + + def submit(self, function, *args, **kwargs): + result = Future() + self.jobs.append((function, args, kwargs, result)) + return result + + def drain(self): + while self.jobs: + function, args, kwargs, result = self.jobs.pop(0) + try: + result.set_result(function(*args, **kwargs)) + except Exception as exc: + result.set_exception(exc) + raise + + +@pytest.fixture +def lifecycle_environment(knowledge_environment, monkeypatch): + from sqlalchemy.orm import scoped_session, sessionmaker + + database, client, chat_id = knowledge_environment + queue = QueuedExecutor() + maker = scoped_session(sessionmaker(bind=database)) + monkeypatch.setattr(embedding_threads, "executor", queue) + monkeypatch.setattr(embedding_threads, "session_maker", maker) + yield database, client, chat_id, queue + maker.remove() + + +def test_disabled_knowledge_keeps_literal_recall_and_invalidates_edited_inputs( + lifecycle_environment, +): + database, client, chat_id, queue = lifecycle_environment + with Session(database) as session: + old_task = question_task(session, chat_id) + before = recall(old_task, session) + assert "term-alpha" in before[0] + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code == 200 + ) + assert values(client)["embedding.enabled"] == "false" + with Session(database) as session: + assert recall(old_task, session) == before + result = recall(question_task(session, chat_id), session) + assert "alpha" not in "".join(result) + assert "literal" in result[0] and "literal" in result[1] + term = session.exec( + select(Terminology).where( + Terminology.word == "alpha", + Terminology.oid == 1, + Terminology.advanced_application.is_(None), + ) + ).first() + example = session.exec( + select(DataTraining).where( + DataTraining.question == "alpha", + DataTraining.oid == 1, + DataTraining.datasource == 10, + ) + ).first() + term_id, example_id = term.id, example.id + update_terminology( + session, + TerminologyInfo( + id=term_id, + word="new term", + description="edited", + other_words=["new synonym"], + specific_ds=True, + datasource_ids=[10], + enabled=True, + ), + 1, + lambda key: key, + ) + update_training( + session, + DataTrainingInfo( + id=example_id, + question="new question", + description="select 1", + datasource=10, + enabled=True, + ), + 1, + lambda key: key, + ) + queue.drain() + with Session(database) as session: + assert session.get(Terminology, term_id).embedding is None + assert session.get(DataTraining, example_id).embedding is None + synonym = session.exec( + select(Terminology).where(Terminology.pid == term_id) + ).one() + assert synonym.embedding is None + + +def test_reenable_fills_missing_vectors_and_keeps_valid_vectors( + lifecycle_environment, monkeypatch +): + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.data_training.curd.data_training import create_training + from apps.terminology.curd.terminology import create_terminology + + database, client, _chat_id, queue = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code == 200 + ) + with Session(database) as session: + term_id = create_terminology( + session, + TerminologyInfo( + word="missing term", + description="meaning", + other_words=["missing synonym"], + specific_ds=True, + datasource_ids=[10], + ), + 1, + lambda key: key, + ) + example_id = create_training( + session, + DataTrainingInfo( + question="missing question", description="select 1", datasource=10 + ), + 1, + lambda key: key, + ) + queue.drain() + generated = [] + + def embed_documents(words): + generated.extend(words) + return [[0.6, 0.8] for _word in words] + + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_documents=embed_documents), + ) + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": True}]).status_code == 200 + ) + # Saving only queues work: a slow embedding provider cannot block the API. + assert generated == [] + queue.drain() + with Session(database) as session: + assert list(session.get(Terminology, term_id).embedding) == pytest.approx( + [0.6, 0.8] + ) + assert list(session.get(DataTraining, example_id).embedding) == pytest.approx( + [0.6, 0.8] + ) + assert list( + session.exec(select(Terminology).where(Terminology.pid == term_id)) + .one() + .embedding + ) == pytest.approx([0.6, 0.8]) + assert sorted(generated) == ["missing question", "missing synonym", "missing term"] + + +@pytest.mark.parametrize("kind", ["terminology", "data_training"]) +@pytest.mark.parametrize("change", ["edit", "delete", "unchanged"]) +def test_in_flight_old_content_cannot_restore_invalidated_or_deleted_vectors( + lifecycle_environment, + monkeypatch, + kind, + change, +): + import threading + from concurrent.futures import ThreadPoolExecutor + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.data_training.curd.data_training import create_training, delete_training + from apps.terminology.curd.terminology import create_terminology, delete_terminology + + database, client, _chat_id, queue = lifecycle_environment + entered, release = threading.Event(), threading.Event() + + def embed_documents(words): + entered.set() + assert release.wait(10), "test did not release provider" + return [[1, 0] for _ in words] + + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_documents=embed_documents), + ) + with Session(database) as session: + if kind == "terminology": + resource_id = create_terminology( + session, + TerminologyInfo( + word="old term", + description="meaning", + other_words=["old synonym"], + specific_ds=True, + datasource_ids=[10], + ), + 1, + lambda key: key, + ) + model = Terminology + else: + resource_id = create_training( + session, + DataTrainingInfo( + question="old question", description="select 1", datasource=10 + ), + 1, + lambda key: key, + ) + model = DataTraining + function, args, kwargs, _result = queue.jobs.pop(0) + with ThreadPoolExecutor(max_workers=1) as pool: + task = pool.submit(function, *args, **kwargs) + try: + assert entered.wait(10), "generation never reached provider" + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code + == 200 + ) + with Session(database) as session: + if change == "delete": + (delete_terminology if kind == "terminology" else delete_training)( + session, [resource_id], 1 + ) + elif change == "edit" and kind == "terminology": + update_terminology( + session, + TerminologyInfo( + id=resource_id, + word="new term", + description="meaning", + other_words=["new synonym"], + specific_ds=True, + datasource_ids=[10], + ), + 1, + lambda key: key, + ) + elif change == "edit": + update_training( + session, + DataTrainingInfo( + id=resource_id, + question="new question", + description="select 1", + datasource=10, + ), + 1, + lambda key: key, + ) + finally: + release.set() + task.result(timeout=10) + queue.drain() + with Session(database) as session: + resource = session.get(model, resource_id) + if change == "delete": + assert resource is None + elif change == "unchanged": + assert list(resource.embedding) == [1, 0] + else: + assert resource.embedding is None + + +def test_concurrent_startup_and_resource_job_generate_each_input_once( + lifecycle_environment, monkeypatch +): + import threading + from concurrent.futures import ThreadPoolExecutor + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.data_training.curd.data_training import create_training + + database, _client, _chat_id, queue = lifecycle_environment + entered, release = threading.Event(), threading.Event() + import time + import uuid + + from sqlalchemy import event, text + + application = "knowledge-concurrency-" + uuid.uuid4().hex + + def identify_connection(connection, _record, _proxy): + connection.execute( + "SELECT set_config('application_name', %s, false)", (application,) + ) + connection.commit() + + event.listen(database, "checkout", identify_connection) + duplicate = threading.Event() + generated = [] + + def embed_documents(words): + generated.extend(words) + if len(generated) > 1: + duplicate.set() + entered.set() + assert release.wait(10) + return [[1, 0] for _ in words] + + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_documents=embed_documents), + ) + with Session(database) as session: + create_training( + session, + DataTrainingInfo( + question="concurrent question", description="select 1", datasource=10 + ), + 1, + lambda key: key, + ) + embedding_threads.fill_empty_data_training_embeddings() + with ThreadPoolExecutor(max_workers=2) as pool: + tasks = [ + pool.submit(function, *args, **kwargs) + for function, args, kwargs, _ in queue.jobs + ] + try: + assert entered.wait(10) + deadline = time.monotonic() + 10 + while not duplicate.is_set(): + with database.connect() as connection: + waiting = connection.execute( + text( + "SELECT count(*) FROM pg_stat_activity WHERE application_name = :name AND wait_event = 'advisory'" + ), + {"name": application}, + ).scalar_one() + if waiting: + break + assert time.monotonic() < deadline, ( + "neither duplicate provider nor lock contention observed" + ) + duplicate.wait(0.01) + finally: + release.set() + for task in tasks: + task.result(timeout=10) + assert generated == ["concurrent question"] + + +def test_synonym_edit_preserves_unchanged_vectors(lifecycle_environment, monkeypatch): + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.terminology.curd.terminology import create_terminology + + database, client, _chat_id, queue = lifecycle_environment + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_documents=lambda words: [[1, 0] for _ in words]), + ) + with Session(database) as session: + resource_id = create_terminology( + session, + TerminologyInfo( + word="main", + description="meaning", + other_words=["keep", "remove"], + specific_ds=True, + datasource_ids=[10], + ), + 1, + lambda key: key, + ) + queue.drain() + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code == 200 + ) + with Session(database) as session: + update_terminology( + session, + TerminologyInfo( + id=resource_id, + word="main", + description="new meaning", + other_words=["keep", "added"], + specific_ds=True, + datasource_ids=[10], + ), + 1, + lambda key: key, + ) + queue.drain() + with Session(database) as session: + rows = session.exec( + select(Terminology).where( + (Terminology.id == resource_id) | (Terminology.pid == resource_id) + ) + ).all() + vectors = {row.word: row.embedding for row in rows} + assert set(vectors) == {"main", "keep", "added"} + assert list(vectors["main"]) == [1, 0] + assert list(vectors["keep"]) == [1, 0] + assert vectors["added"] is None + + +@pytest.mark.parametrize("other_words", [None, [], [" "]]) +def test_update_terminology_can_clear_all_synonyms(lifecycle_environment, other_words): + from apps.terminology.curd.terminology import create_terminology + + database, client, _chat_id, queue = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code == 200 + ) + with Session(database) as session: + resource_id = create_terminology( + session, + TerminologyInfo(word="main", description="meaning", other_words=["alias"]), + 1, + lambda key: key, + ) + assert ( + update_terminology( + session, + TerminologyInfo( + id=resource_id, + word="main", + description="updated", + other_words=other_words, + ), + 1, + lambda key: key, + ) + == resource_id + ) + queue.drain() + with Session(database) as session: + assert session.get(Terminology, resource_id).description == "updated" + assert ( + session.exec( + select(Terminology).where(Terminology.pid == resource_id) + ).all() + == [] + ) + + +@pytest.mark.parametrize("crash_first_worker", [False, True]) +def test_processes_share_fill_locks_and_restart_recovers_pending_work( + lifecycle_environment, + tmp_path, + crash_first_worker, +): + import json + import subprocess + import sys + import time + import uuid + + from sqlalchemy import text + + from apps.data_training.curd.data_training import create_training + from apps.terminology.curd.terminology import create_terminology + + database, client, _chat_id, queue = lifecycle_environment + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code == 200 + ) + with Session(database) as session: + term_id = create_terminology( + session, + TerminologyInfo( + word="process term", + description="meaning", + specific_ds=True, + datasource_ids=[10], + ), + 1, + lambda key: key, + ) + example_id = create_training( + session, + DataTrainingInfo( + question="process question", description="select 1", datasource=10 + ), + 1, + lambda key: key, + ) + schema = session.execute(text("SELECT current_schema()")).scalar_one() + queue.drain() + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": True}]).status_code == 200 + ) + application = "knowledge-process-" + uuid.uuid4().hex + worker = tmp_path / "worker.py" + worker.write_text(""" +import json, sys, time +from pathlib import Path +from types import SimpleNamespace +from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +from apps.ai_model.embedding import EmbeddingModelCache +from common.core.config import settings +from common.utils import embedding_threads +url, schema, application, directory = sys.argv[1:] +directory = Path(directory) +settings.EMBEDDING_ENABLED = False +engine = create_engine(url, connect_args={"options": "-csearch_path=" + schema + ",public", "application_name": application}) +embedding_threads.session_maker = scoped_session(sessionmaker(bind=engine)) +def embed_documents(words): + with (directory / "generated.jsonl").open("a") as output: + output.write(json.dumps(words) + "\\n") + deadline = time.monotonic() + 30 + while not (directory / "release").exists(): + if time.monotonic() >= deadline: + raise TimeoutError("provider barrier was not released") + time.sleep(0.01) + return [[0.6, 0.8] for _ in words] +EmbeddingModelCache.get_model = staticmethod(lambda: SimpleNamespace(embed_documents=embed_documents)) +embedding_threads.fill_empty_terminology_embeddings().result(timeout=35) +embedding_threads.fill_empty_data_training_embeddings().result(timeout=35) +embedding_threads.executor.shutdown(wait=True) +engine.dispose() +print("completed") +""") + command = [ + sys.executable, + str(worker), + database.url.render_as_string(hide_password=False), + schema, + application, + str(tmp_path), + ] + processes = [] + + def await_state(predicate, message): + deadline = time.monotonic() + 25 + while not predicate(): + assert time.monotonic() < deadline, message + for process in processes: + assert process.poll() is None, process.communicate()[1] + time.sleep(0.01) + + try: + first = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + processes.append(first) + await_state( + lambda: (tmp_path / "generated.jsonl").exists(), + "first worker did not reach provider", + ) + second = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + processes.append(second) + + def second_is_waiting(): + with database.connect() as connection: + return ( + connection.execute( + text( + "SELECT count(*) FROM pg_stat_activity WHERE application_name=:name AND wait_event='advisory'" + ), + {"name": application}, + ).scalar_one() + == 1 + ) + + await_state( + second_is_waiting, "second process did not wait on shared resource lock" + ) + if crash_first_worker: + first.kill() + first.communicate(timeout=10) + processes.remove(first) + await_state( + lambda: ( + len((tmp_path / "generated.jsonl").read_text().splitlines()) == 2 + ), + "surviving worker did not recover", + ) + (tmp_path / "release").touch() + for process in processes: + stdout, stderr = process.communicate(timeout=35) + assert process.returncode == 0, stderr + assert "completed" in stdout + finally: + (tmp_path / "release").touch() + for process in processes: + if process.poll() is None: + process.kill() + process.communicate(timeout=10) + generated = [ + word + for line in (tmp_path / "generated.jsonl").read_text().splitlines() + for word in json.loads(line) + ] + assert generated.count("process term") == (2 if crash_first_worker else 1) + assert generated.count("process question") == 1 + with Session(database) as session: + assert list(session.get(Terminology, term_id).embedding) == pytest.approx( + [0.6, 0.8] + ) + assert list(session.get(DataTraining, example_id).embedding) == pytest.approx( + [0.6, 0.8] + ) + + +def test_disabled_batch_import_and_reenable_preserve_advanced_scope( + lifecycle_environment, monkeypatch +): + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.data_training.curd.data_training import batch_create_training + from apps.datasource.models.datasource import CoreDatasource + from apps.system.models.system_model import AssistantModel + from apps.terminology.curd.terminology import batch_create_terminology + + database, client, chat_id, queue = lifecycle_environment + CoreDatasource.__table__.create(database) + AssistantModel.__table__.create(database) + with Session(database) as session: + session.add( + AssistantModel( + id=99, + oid=1, + name="import application", + type=1, + domain="http://localhost", + ) + ) + session.commit() + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code == 200 + ) + with Session(database) as session: + term = batch_create_terminology( + session, + [ + TerminologyInfo( + word="import main", + description="imported term", + other_words=["query"], + specific_ds=True, + advanced_application_name="import application", + ) + ], + 1, + lambda key: key, + ) + example = batch_create_training( + session, + [ + DataTrainingInfo( + question="query", + description="imported SQL", + advanced_application_name="import application", + ) + ], + 1, + lambda key: key, + ) + assert term["success_count"] == 1, term + assert example["success_count"] == 1, example + queue.drain() + with Session(database) as session: + imported = session.exec( + select(Terminology).where(Terminology.description == "imported term") + ).one() + synonym = session.exec( + select(Terminology).where(Terminology.pid == imported.id) + ).one() + assert imported.embedding is None and synonym.embedding is None + synonym_id = synonym.id + result = recall( + question_task(session, chat_id, SimpleNamespace(type=1, id=99, oid=1)), + session, + ) + assert "imported term" in result[0] and "imported SQL" in result[1] + ordinary = recall(question_task(session, chat_id), session) + assert "imported" not in "".join(ordinary) + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace( + embed_documents=lambda words: [[1, 0] for _ in words], + embed_query=lambda _word: [1, 0], + ), + ) + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": True}]).status_code == 200 + ) + queue.drain() + with Session(database) as session: + assert list(session.get(Terminology, synonym_id).embedding) == [1, 0] + ordinary = recall(question_task(session, chat_id), session) + assert "imported" not in "".join(ordinary) + + +def test_new_analysis_uses_disabled_snapshot_and_historical_literal_terms( + lifecycle_environment, +): + from types import SimpleNamespace + + from langchain_core.messages import AIMessageChunk + from sqlalchemy import update + + from apps.chat.curd.chat import get_chat_with_records + from apps.chat.models.chat_model import ChatRecord + + database, client, chat_id, _queue = lifecycle_environment + with Session(database) as session: + session.add( + Terminology( + word="literal", + description="analysis literal", + oid=1, + embedding=[0, 1], + specific_ds=False, + ) + ) + session.add( + Terminology( + word="semantic", + description="analysis vector", + oid=1, + embedding=[1, 0], + specific_ds=False, + ) + ) + session.commit() + old_task = question_task(session, chat_id) + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code + == 200 + ) + new_task = question_task(session, chat_id) + messages = [] + + def stream(value): + messages.extend(value) + return iter([AIMessageChunk(content="knowledge analysis")]) + + for task in (old_task, new_task): + task.record.extracted_keywords = "literal" + task.chat_question.question = "unrelated original question" + session.execute( + update(ChatRecord) + .where(ChatRecord.id == task.record.id) + .values(extracted_keywords="literal") + ) + session.commit() + task.llm = SimpleNamespace(stream=stream) + list(task.generate_analysis(session)) + assert "analysis vector" in old_task.chat_question.terminologies + assert "analysis vector" not in new_task.chat_question.terminologies + assert "analysis literal" in new_task.chat_question.terminologies + assert any("analysis literal" in str(message.content) for message in messages) + records = get_chat_with_records( + session, chat_id, new_task.current_user, None + ).records + assert any(record["analysis"] == "knowledge analysis" for record in records) + + +@pytest.mark.parametrize("query_provider_fails", [False, True]) +def test_failed_generation_keeps_literal_and_available_vector_recall( + lifecycle_environment, + monkeypatch, + query_provider_fails, +): + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.data_training.curd.data_training import create_training + from common.utils.embedding_lifecycle import EmbeddingGenerationError + + database, _client, chat_id, queue = lifecycle_environment + + def fail(_value): + raise ConnectionError("controlled provider failure") + + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace( + embed_documents=fail, + embed_query=fail if query_provider_fails else lambda _value: [1, 0], + ), + ) + with Session(database) as session: + resource_id = create_training( + session, + DataTrainingInfo( + question="pending question", description="pending SQL", datasource=10 + ), + 1, + lambda key: key, + ) + with pytest.raises(EmbeddingGenerationError): + queue.drain() + with Session(database) as session: + assert session.get(DataTraining, resource_id).embedding is None + result = recall(question_task(session, chat_id), session) + assert "literal" in result[0] and "literal" in result[1] + assert ("alpha" in result[0]) is not query_provider_fails + assert ("alpha" in result[1]) is not query_provider_fails + + +def test_off_then_on_during_old_generation_fills_latest_input( + lifecycle_environment, monkeypatch +): + import threading + from concurrent.futures import ThreadPoolExecutor + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.data_training.curd.data_training import create_training + + database, client, _chat_id, queue = lifecycle_environment + entered, release = threading.Event(), threading.Event() + generated = [] + + def embed_documents(words): + generated.extend(words) + if "old input" in words: + entered.set() + assert release.wait(10) + return [[1, 0] if word == "old input" else [0, 1] for word in words] + + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_documents=embed_documents), + ) + with Session(database) as session: + resource_id = create_training( + session, + DataTrainingInfo( + question="old input", description="select 1", datasource=10 + ), + 1, + lambda key: key, + ) + function, args, kwargs, _future = queue.jobs.pop(0) + with ThreadPoolExecutor(max_workers=3) as pool: + old_task = pool.submit(function, *args, **kwargs) + try: + assert entered.wait(10) + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": False}]).status_code + == 200 + ) + with Session(database) as session: + update_training( + session, + DataTrainingInfo( + id=resource_id, + question="new input", + description="select 2", + datasource=10, + ), + 1, + lambda key: key, + ) + queue.drain() # The edit job really starts while the switch is off. + assert ( + save(client, [{"pkey": "embedding.enabled", "pval": True}]).status_code + == 200 + ) + tasks = [ + pool.submit(function, *args, **kwargs) + for function, args, kwargs, _future in queue.jobs + ] + finally: + release.set() + old_task.result(timeout=10) + for task in tasks: + task.result(timeout=10) + with Session(database) as session: + assert list(session.get(DataTraining, resource_id).embedding) == [0, 1] + assert generated == ["old input", "new input"] + + +@pytest.mark.parametrize("vector", [[0, 0], [float("nan"), 1], []]) +def test_unusable_query_vector_falls_back_to_literal_knowledge( + lifecycle_environment, monkeypatch, vector +): + from types import SimpleNamespace + + from apps.ai_model.embedding import EmbeddingModelCache + + database, _client, chat_id, _queue = lifecycle_environment + monkeypatch.setattr( + EmbeddingModelCache, + "get_model", + lambda: SimpleNamespace(embed_query=lambda _question: vector), + ) + with Session(database) as session: + result = recall(question_task(session, chat_id), session) + assert "literal" in result[0] and "literal" in result[1] + assert "alpha" not in "".join(result) diff --git a/backend/tests/test_mcp_parameters_integration.py b/backend/tests/test_mcp_parameters_integration.py new file mode 100644 index 00000000..c9e56bb4 --- /dev/null +++ b/backend/tests/test_mcp_parameters_integration.py @@ -0,0 +1,276 @@ +"""Opt-in independent Uvicorn/MCP SSE acceptance on an isolated PostgreSQL schema. + +Only LLM, embedding and external datasource providers are controlled. Authentication, +MCP transport, parameter API, task creation, retrieval and SQL validation stay real. +API lifespan migrations are disabled because the fixture owns its schema; the MCP +process uses the production main:mcp_app with its normal lifespan. +""" + +import asyncio +import json +import logging +import os +import secrets +import socket +import subprocess +import sys +import time +from contextlib import contextmanager +from pathlib import Path + +import httpx +import test_table_parameters_integration as table_contracts +from mcp import ClientSession +from mcp.client.sse import sse_client +from sqlalchemy import text +from sqlmodel import Session +from test_embedding_parameters_integration import ( # noqa: F401 + knowledge_environment, + pytestmark, +) + +from apps.system.models.system_model import AiModelDetail, UserWsModel +from apps.system.models.user import UserModel +from common.core.security import md5pwd + +table_environment = table_contracts.table_environment + + +def serve_controlled_process(app_name, port): + """Child entry point: substitute providers before Uvicorn imports main.""" + from types import SimpleNamespace + + import uvicorn + from langchain_core.messages import AIMessageChunk + + from apps.ai_model.embedding import EmbeddingModelCache + from apps.ai_model.model_factory import BaseLLM, LLMFactory + from apps.chat.task import llm + + class Provider(BaseLLM): + def _init_llm(self): + return self + + def stream(self, messages): + context = "\n".join(message.content for message in messages) + with Path(os.environ["SQLBOT_TEST_CONTEXTS"]).open("a") as output: + output.write(json.dumps(context) + "\n") + # Chart prompts include the SQL that has just passed validation. + if "SELECT id FROM vector_beta" in messages[-1].content: + answer = {"type": "table", "columns": [{"name": "id", "value": "id"}]} + else: + answer = { + "success": True, + "sql": "SELECT id FROM vector_beta", + "tables": ["vector_beta"], + } + yield AIMessageChunk(content=json.dumps(answer)) + + LLMFactory.register_llm("openai", Provider) + EmbeddingModelCache.get_model = staticmethod( + lambda: SimpleNamespace(embed_query=lambda _text: [1.0, 0.0]) + ) + llm.get_version = lambda _ds: "" + llm.check_connection = lambda **_kwargs: True + llm.exec_sql = lambda **_kwargs: {"fields": ["id"], "data": [{"id": 7}]} + uvicorn.run( + "main:" + app_name, + host="127.0.0.1", + port=port, + lifespan="off" if app_name == "app" else "on", + log_level="warning", + ) + + +@contextmanager +def server(app_name, env, directory): + with socket.socket() as bound: + bound.bind(("127.0.0.1", 0)) + port = bound.getsockname()[1] + log_path = directory / (app_name + ".log") + with log_path.open("w") as log: + process = subprocess.Popen( + [ + sys.executable, + "-c", + "from test_mcp_parameters_integration import serve_controlled_process; " + f"serve_controlled_process({app_name!r}, {port})", + ], + cwd=Path(__file__).parents[1], + env=env, + stdout=log, + stderr=log, + ) + try: + deadline = time.monotonic() + 40 + while True: + assert process.poll() is None, log_path.read_text() + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.1): + break + except OSError: + assert time.monotonic() < deadline, log_path.read_text() + time.sleep(0.05) + yield f"http://127.0.0.1:{port}" + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def test_mcp_network_task_uses_parameter_saved_by_another_api_process( + table_environment, tmp_path, caplog +): + # MCP debug messages contain request bodies, including this disposable token. + caplog.set_level(logging.WARNING, logger="mcp.client.sse") + database, _client, ds_id = table_environment + for model in (UserModel, UserWsModel, AiModelDetail): + model.__table__.create(database) + password = secrets.token_urlsafe(24) + with Session(database) as session: + schema = session.execute(text("SELECT current_schema()")).scalar_one() + session.add( + UserModel( + id=1, + account="admin", + name="MCP test", + oid=1, + email="mcp@example.invalid", + password=md5pwd(password), + status=1, + ) + ) + session.add(UserWsModel(uid=1, oid=1, weight=1)) + session.add( + AiModelDetail( + supplier=1, + name="controlled", + model_type=1, + base_model="controlled", + default_model=True, + api_domain="http://127.0.0.1:9/v1", + api_key="unused", + config="[]", + ) + ) + session.commit() + for name in ("logs", "images", "scripts", "files", "excel", "models"): + (tmp_path / name).mkdir() + env = { + **os.environ, + "SQLBOT_DB_URL": database.url.update_query_dict( + {"options": f"-csearch_path={schema},public"} + ).render_as_string(hide_password=False), + "PYTHONPATH": os.pathsep.join( + [str(Path(__file__).parent), str(Path(__file__).parents[1])] + ), + "SECRET_KEY": secrets.token_urlsafe(48), + "CACHE_TYPE": "None", + "CONTEXT_PATH": "", + "BASE_DIR": str(tmp_path), + "LOG_DIR": str(tmp_path / "logs"), + "MCP_IMAGE_PATH": str(tmp_path / "images"), + "SCRIPT_DIR": str(tmp_path / "scripts"), + "UPLOAD_DIR": str(tmp_path / "files"), + "EXCEL_PATH": str(tmp_path / "excel"), + "LOCAL_MODEL_PATH": str(tmp_path / "models"), + "LOG_FORMAT": "%(levelname)s %(message)s", + "LOG_LEVEL": "WARNING", + "SQLBOT_TEST_CONTEXTS": str(tmp_path / "contexts.jsonl"), + "EMBEDDING_ENABLED": "false", + "TABLE_EMBEDDING_ENABLED": "true", + "TABLE_EMBEDDING_COUNT": "10", + "TABLE_EMBEDDING_KEYWORD_ENABLED": "false", + "SAMPLE_DATA_ENABLED": "false", + } + with ( + server("app", env, tmp_path) as api, + server("mcp_app", env, tmp_path) as mcp_url, + ): + with httpx.Client(base_url=api, timeout=15) as client: + response = client.post( + "/api/v1/mcp/access_token", + json={"username": "admin", "password": password}, + ) + assert response.status_code == 200, response.text + token = response.json()["data"]["access_token"] + client.headers["X-SQLBOT-TOKEN"] = "Bearer " + token + + async def ask_twice(): + async with sse_client( + mcp_url + "/mcp", timeout=15, sse_read_timeout=30 + ) as streams: + async with ClientSession(*streams) as session: + await session.initialize() + assert "mcp_question" in { + tool.name for tool in (await session.list_tools()).tools + } + rejected = await session.call_tool( + "mcp_question", + { + "token": "invalid", + "chat_id": 0, + "question": "show totals", + "stream": False, + }, + ) + assert rejected.isError + for count in (1, 2): + response = client.post( + "/api/v1/system/parameter", + data={ + "data": json.dumps( + [ + { + "pkey": "embedding.table_count", + "pval": count, + }, + ] + ) + }, + ) + assert response.status_code == 200, response.text + started = await session.call_tool( + "mcp_start", {"token": token} + ) + assert not started.isError, started + chat_id = json.loads(started.content[0].text)["data"][ + "chat_id" + ] + result = await session.call_tool( + "mcp_question", + { + "token": token, + "chat_id": chat_id, + "datasource_id": str(ds_id), + "question": "show totals", + "stream": False, + "return_img": False, + "lang": "en", + }, + ) + assert not result.isError, result + answer = json.loads(result.content[0].text) + assert ( + answer["success"] + and answer["sql"] == "SELECT id FROM vector_beta" + ) + assert answer["data"] + contexts = [ + json.loads(line) + for line in (tmp_path / "contexts.jsonl") + .read_text() + .splitlines() + ] + sql_context = contexts[-2] + assert "# Table: synthetic.vector_beta" in sql_context + assert ( + "# Table: synthetic.other_gamma" in sql_context + ) == (count == 2) + assert "# Table: synthetic.hidden_table" not in sql_context + assert "# Table: synthetic.sales_alpha" not in sql_context + + asyncio.run(ask_twice()) diff --git a/backend/tests/test_table_embedding.py b/backend/tests/test_table_embedding.py index b2b149d0..06175dc1 100644 --- a/backend/tests/test_table_embedding.py +++ b/backend/tests/test_table_embedding.py @@ -662,7 +662,7 @@ def test_vector_failure_does_not_remove_explicit_table(self, failure): else: tables[-1]['embedding'] = None result = calc_table_embedding(tables, f'查询{self.COMMENT}', keywords='采购') - assert len(result) == (10 if failure == 'missing_embedding' else len(tables)) + assert len(result) == len(tables) assert result[0]['table_name'] == self.TARGET @pytest.mark.parametrize('keywords', [None, '销售额']) diff --git a/backend/tests/test_table_parameters_integration.py b/backend/tests/test_table_parameters_integration.py new file mode 100644 index 00000000..44b640c2 --- /dev/null +++ b/backend/tests/test_table_parameters_integration.py @@ -0,0 +1,632 @@ +"""Opt-in task contracts using isolated PG; only model/source providers substituted.""" + +import json +from datetime import datetime +from types import SimpleNamespace + +import pytest +from langchain_core.messages import AIMessageChunk +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlbot_xpack.permissions.models.ds_rules import DsRules +from sqlmodel import Session +from test_embedding_parameters import save +from test_embedding_parameters_integration import ( # noqa: F401 + knowledge_environment, + pytestmark, +) + +from apps.ai_model.model_factory import LLMConfig +from apps.chat.models.chat_model import Chat, ChatFinishStep, ChatQuestion, ChatRecord +from apps.chat.task.llm import LLMService +from apps.datasource.models.datasource import CoreDatasource, CoreField, CoreTable +from apps.datasource.utils.utils import aes_encrypt +from apps.terminology.models.terminology_model import Terminology +from common.core.config import settings + + +@pytest.fixture +def table_environment(knowledge_environment, monkeypatch): # noqa: F811 + database, client, _chat_id = knowledge_environment + for model in (CoreDatasource, CoreTable, CoreField, DsRules): + model.__table__.create(database) + monkeypatch.setattr(settings, "TABLE_EMBEDDING_ENABLED", True) + monkeypatch.setattr(settings, "TABLE_EMBEDDING_COUNT", 10) + monkeypatch.setattr(settings, "TABLE_EMBEDDING_KEYWORD_ENABLED", False) + monkeypatch.setattr(settings, "SAMPLE_DATA_ENABLED", False) + monkeypatch.setattr("apps.chat.task.llm.get_version", lambda _ds: "") + monkeypatch.setattr("apps.chat.task.llm.check_connection", lambda **_kw: True) + sessions = scoped_session(sessionmaker(bind=database, class_=Session)) + monkeypatch.setattr("apps.chat.task.llm.session_maker", sessions) + with Session(database) as session: + ds = CoreDatasource( + name="accessible", + description="test", + type="pg", + type_name="PostgreSQL", + configuration=aes_encrypt('{"database":"synthetic"}').decode(), + oid=1, + embedding="[1,0]", + ) + session.add(ds) + session.flush() + ds_id = ds.id + for name, comment, vector, checked in [ + ("sales_alpha", "keyword winner", "[0,1]", True), + ("vector_beta", "vector winner", "[1,0]", True), + ("other_gamma", "other", "[0.8,0.6]", True), + ("hidden_table", "sales", "[1,0]", False), + ]: + table = CoreTable( + ds_id=ds_id, + table_name=name, + table_comment=comment, + embedding=vector, + checked=checked, + ) + session.add(table) + session.flush() + session.add( + CoreField( + ds_id=ds_id, + table_id=table.id, + field_name="id", + field_type="INTEGER", + field_index=1, + ) + ) + session.commit() + yield database, client, ds_id + sessions.remove() + + +class QuestionProvider: + def __init__(self, *, datasource=None, on_datasource=None): + self.datasource = datasource + self.on_datasource = on_datasource + self.contexts = [] + + def stream(self, messages): + context = "\n".join(message.content for message in messages) + self.contexts.append(context) + if "【Schema】" in context: + table = ( + "vector_beta" + if "# Table: synthetic.vector_beta" in context + else "sales_alpha" + ) + content = json.dumps( + {"success": True, "sql": f"SELECT id FROM {table}", "tables": [table]} + ) + elif self.datasource is not None and self.datasource["name"] in context: + if self.on_datasource: + self.on_datasource() + content = json.dumps({"id": self.datasource["id"]}) + else: + content = "sales" + yield AIMessageChunk(content=content) + + +def task(database, ds_id, *, question="show totals", assistant=None, user_id=1): + with Session(database) as session: + chat = Chat( + oid=1, + create_by=1, + create_time=datetime.now(), + brief="test", + datasource=ds_id, + engine_type="PostgreSQL", + ) + session.add(chat) + session.commit() + service = LLMService( + session, + SimpleNamespace(id=user_id, oid=1, language="en"), + ChatQuestion(chat_id=chat.id, question=question), + current_assistant=assistant, + config=LLMConfig( + model_type="openai", + model_name="test", + api_key="test", + api_base_url="http://127.0.0.1:9/v1", + ), + ) + service.init_record(session) + service.llm = QuestionProvider() + return service + + +def run(service): + results = list( + service.run_task( + in_chat=False, stream=False, finish_step=ChatFinishStep.GENERATE_SQL + ) + ) + assert results[-1]["success"], results + return service.chat_question.db_schema + + +def configure(client, **parameters): + response = save( + client, + [ + {"pkey": f"embedding.{key}", "pval": value} + for key, value in parameters.items() + ], + ) + assert response.status_code == 200, response.text + + +def test_question_table_count_and_in_flight_snapshot(table_environment): + database, client, ds_id = table_environment + configure(client, table_count=1) + old = task(database, ds_id) + configure(client, table_count=2) + old_schema = run(old) + assert "vector_beta" in old_schema + assert "sales_alpha" not in old_schema and "other_gamma" not in old_schema + current = task(database, ds_id) + schema = run(current) + assert "vector_beta" in schema and "other_gamma" in schema + assert "hidden_table" not in schema and "sales_alpha" not in schema + assert any(schema in context for context in current.llm.contexts) + with Session(database) as session: + assert ( + session.get(ChatRecord, current.record.id).sql + == "SELECT id FROM vector_beta" + ) + + +def test_keyword_switch_and_weight_change_question_context(table_environment): + database, client, ds_id = table_environment + with Session(database) as session: + term = Terminology( + oid=1, word="sales", description="keyword-only-term", embedding=[0, 1] + ) + session.add(term) + session.flush() + session.add(Terminology(oid=1, pid=term.id, word="revenue", embedding=[0, 1])) + session.commit() + configure(client, table_count=1, table_alpha=0, keyword_enabled=True) + keyword_task = task(database, ds_id) + configure(client, keyword_enabled=False) + schema = run(keyword_task) + assert "sales_alpha" in schema and "vector_beta" not in schema + with Session(database) as session: + record = session.get(ChatRecord, keyword_task.record.id) + assert record.extracted_keywords == "sales" + assert "revenue" in record.expanded_keywords + assert "keyword-only-term" in keyword_task.chat_question.terminologies + disabled = task(database, ds_id) + schema = run(disabled) + assert "vector_beta" in schema and "sales_alpha" not in schema + with Session(database) as session: + assert not session.get(ChatRecord, disabled.record.id).extracted_keywords + assert "keyword-only-term" not in disabled.chat_question.terminologies + configure(client, keyword_enabled=True, table_alpha=1) + schema = run(task(database, ds_id)) + assert "vector_beta" in schema and "sales_alpha" not in schema + + +def test_keywords_remain_independent_of_table_vectors(table_environment, monkeypatch): + database, client, ds_id = table_environment + monkeypatch.setattr(settings, "TABLE_EMBEDDING_ENABLED", False) + configure(client, keyword_enabled=True, table_count=1) + current = task(database, ds_id) + schema = run(current) + assert all(name in schema for name in ("sales_alpha", "vector_beta", "other_gamma")) + with Session(database) as session: + assert session.get(ChatRecord, current.record.id).extracted_keywords == "sales" + + +@pytest.mark.parametrize( + "bad_vector", [None, "[]", "broken", "[0,0]", "[NaN,1]", "[1,0,0]"] +) +def test_incomplete_table_vectors_preserve_all_eligible_context( + table_environment, bad_vector +): + database, client, ds_id = table_environment + configure(client, table_count=1) + with Session(database) as session: + table = ( + session.query(CoreTable).filter(CoreTable.table_name == "sales_alpha").one() + ) + table.embedding = bad_vector + session.add(table) + session.commit() + schema = run(task(database, ds_id)) + assert all(name in schema for name in ("sales_alpha", "vector_beta", "other_gamma")) + assert "hidden_table" not in schema + + +def test_failed_query_vectors_preserve_all_eligible_context( + table_environment, monkeypatch +): + from apps.ai_model.embedding import EmbeddingModelCache + + database, client, ds_id = table_environment + configure(client, table_count=1, keyword_enabled=True) + + def fail(_query): + raise RuntimeError("controlled embedding failure") + + monkeypatch.setattr( + EmbeddingModelCache, "get_model", lambda: SimpleNamespace(embed_query=fail) + ) + # Knowledge vectors are separate from this test; keep literal knowledge active. + monkeypatch.setattr(settings, "EMBEDDING_ENABLED", False) + schema = run(task(database, ds_id)) + assert all(name in schema for name in ("sales_alpha", "vector_beta", "other_gamma")) + assert "hidden_table" not in schema + + +def add_datasource_candidates(database): + with Session(database) as session: + session.add( + CoreDatasource( + name="second_candidate", + description="test", + type="pg", + oid=1, + embedding="[0,1]", + ) + ) + session.add( + CoreDatasource( + name="foreign_workspace", + description="test", + type="pg", + oid=2, + embedding="[1,0]", + ) + ) + session.commit() + + +def test_datasource_preselection_and_following_steps_share_task_snapshot( + table_environment, +): + database, client, ds_id = table_environment + add_datasource_candidates(database) + configure(client, datasource_count=1, table_count=1, keyword_enabled=False) + old = task(database, None) + old.llm = QuestionProvider( + datasource={"id": ds_id, "name": "accessible"}, + on_datasource=lambda: configure( + client, + datasource_count=2, + table_count=2, + keyword_enabled=True, + table_alpha=0, + ), + ) + schema = run(old) + assert "second_candidate" not in old.llm.contexts[0] + assert "foreign_workspace" not in old.llm.contexts[0] + assert ( + "vector_beta" in schema + and "sales_alpha" not in schema + and "other_gamma" not in schema + ) + current = task(database, None) + current.llm = QuestionProvider(datasource={"id": ds_id, "name": "accessible"}) + schema = run(current) + assert "second_candidate" in current.llm.contexts[0] + assert "foreign_workspace" not in current.llm.contexts[0] + assert "sales_alpha" in schema + with Session(database) as session: + assert not session.get(ChatRecord, old.record.id).extracted_keywords + assert session.get(ChatRecord, current.record.id).extracted_keywords == "sales" + + +@pytest.mark.parametrize( + "bad_vector", [None, "[]", "broken", "[0,0]", "[NaN,1]", "[1,0,0]"] +) +def test_incomplete_datasource_vectors_keep_original_candidate_dtos( + table_environment, bad_vector +): + database, client, ds_id = table_environment + add_datasource_candidates(database) + configure(client, datasource_count=1) + with Session(database) as session: + ds = ( + session.query(CoreDatasource) + .filter(CoreDatasource.name == "second_candidate") + .one() + ) + ds.embedding = bad_vector + session.add(ds) + session.commit() + current = task(database, None) + current.llm = QuestionProvider(datasource={"id": ds_id, "name": "accessible"}) + run(current) + assert "second_candidate" in current.llm.contexts[0] + assert "foreign_workspace" not in current.llm.contexts[0] + assert "configuration" not in current.llm.contexts[0] + + +def test_explicit_tables_and_relations_preserved_without_unchecked_tables( + table_environment, +): + database, client, ds_id = table_environment + configure(client, table_count=1) + with Session(database) as session: + tables = {table.table_name: table for table in session.query(CoreTable).all()} + fields = {field.table_id: field for field in session.query(CoreField).all()} + + def edge(source, target): + return { + "shape": "edge", + "source": { + "cell": str(tables[source].id), + "port": str(fields[tables[source].id].id), + }, + "target": { + "cell": str(tables[target].id), + "port": str(fields[tables[target].id].id), + }, + } + + ds = session.get(CoreDatasource, ds_id) + ds.table_relation = [ + edge("vector_beta", "other_gamma"), + edge("vector_beta", "hidden_table"), + ] + session.add(ds) + session.commit() + schema = run(task(database, ds_id)) + assert "vector_beta" in schema and "other_gamma" in schema + assert "vector_beta.id=other_gamma.id" in schema + assert "hidden_table" not in schema and "sales_alpha" not in schema + schema = run(task(database, ds_id, question="compare sales_alpha and other_gamma")) + assert all(name in schema for name in ("sales_alpha", "other_gamma", "vector_beta")) + assert "hidden_table" not in schema + + +@pytest.mark.parametrize("assistant_type", [0, 2, 4]) +def test_ordinary_and_embedded_tasks_apply_saved_selection( + table_environment, assistant_type +): + database, client, ds_id = table_environment + add_datasource_candidates(database) + configure(client, table_count=1, keyword_enabled=False, datasource_count=1) + assistant = SimpleNamespace( + type=assistant_type, + id=99, + oid=1, + online=False, + configuration=json.dumps({"oid": 1, "public_list": [ds_id]}), + ) + current = task(database, None, assistant=assistant) + current.llm = QuestionProvider(datasource={"id": ds_id, "name": "accessible"}) + schema = run(current) + assert "vector_beta" in schema and "sales_alpha" not in schema + assert "second_candidate" not in "\n".join(current.llm.contexts) + assert "foreign_workspace" not in "\n".join(current.llm.contexts) + + +def test_advanced_application_keeps_full_dynamic_candidates( + table_environment, monkeypatch +): + database, client, _ds_id = table_environment + configure(client, table_count=1, datasource_count=1, keyword_enabled=False) + dynamic = [ + { + "id": index, + "name": name, + "type": "pg", + "dataBase": "synthetic", + "tables": [ + { + "name": table, + "comment": "", + "fields": [{"name": "id", "type": "INTEGER", "comment": ""}], + } + for table in ("sales_alpha", "vector_beta", "other_gamma") + ], + } + for index, name in [(901, "dynamic_first"), (902, "dynamic_second")] + ] + monkeypatch.setattr( + "apps.system.crud.assistant.requests.get", + lambda **_kw: SimpleNamespace( + status_code=200, text=json.dumps({"code": 0, "data": dynamic}) + ), + ) + assistant = SimpleNamespace( + type=1, + id=99, + oid=1, + certificate="[]", + request_origin=None, + configuration=json.dumps({"endpoint": "https://host.invalid/datasources"}), + ) + current = task(database, None, assistant=assistant) + current.llm = QuestionProvider(datasource={"id": 901, "name": "dynamic_first"}) + schema = run(current) + assert "dynamic_second" in current.llm.contexts[0] + assert all(name in schema for name in ("sales_alpha", "vector_beta", "other_gamma")) + + +def test_saved_selection_visible_to_independent_task_processes(table_environment): + import os + import subprocess + import sys + from concurrent.futures import ThreadPoolExecutor + from pathlib import Path + + from sqlalchemy import text + + database, client, ds_id = table_environment + add_datasource_candidates(database) + configure( + client, table_count=1, table_alpha=0, keyword_enabled=True, datasource_count=1 + ) + with database.connect() as connection: + schema = connection.execute(text("select current_schema()")).scalar_one() + script = """ +import json, os +from types import SimpleNamespace +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlmodel import Session, create_engine +import test_table_parameters_integration as contract +from apps.ai_model.embedding import EmbeddingModelCache +import apps.chat.task.llm as tasks +EmbeddingModelCache.get_model = staticmethod(lambda: SimpleNamespace(embed_query=lambda _: [1.0, 0.0])) +db = create_engine(os.environ['SQLBOT_TEST_DATABASE_URL'], connect_args={'options': '-csearch_path=' + os.environ['SQLBOT_TEST_SCHEMA'] + ',public'}) +tasks.session_maker = scoped_session(sessionmaker(bind=db, class_=Session)) +tasks.get_version = lambda _: '' +tasks.check_connection = lambda **_: True +service = contract.task(db, None) +service.llm = contract.QuestionProvider(datasource={'id': int(os.environ['SQLBOT_TEST_DS_ID']), 'name': 'accessible'}) +schema = contract.run(service) +print('RESULT:' + json.dumps({'schema': schema, 'candidates': service.llm.contexts[0]})) +tasks.session_maker.remove() +db.dispose() +""" + env = { + **os.environ, + "SQLBOT_TEST_SCHEMA": schema, + "SQLBOT_TEST_DS_ID": str(ds_id), + "TABLE_EMBEDDING_ENABLED": "true", + "TABLE_EMBEDDING_COUNT": "10", + "TABLE_EMBEDDING_ALPHA": "1", + "TABLE_EMBEDDING_KEYWORD_ENABLED": "false", + "DS_EMBEDDING_COUNT": "10", + "SAMPLE_DATA_ENABLED": "false", + "EMBEDDING_ENABLED": "false", + "PYTHONPATH": os.pathsep.join( + [str(Path(__file__).parent), str(Path(__file__).parents[1])] + ), + } + + def process(_index): + result = subprocess.run( + [sys.executable, "-c", script], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + return json.loads( + next( + line.removeprefix("RESULT:") + for line in result.stdout.splitlines() + if line.startswith("RESULT:") + ) + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(process, range(2))) + for result in results: + assert ( + "sales_alpha" in result["schema"] and "vector_beta" not in result["schema"] + ) + assert "second_candidate" not in result["candidates"] + assert "foreign_workspace" not in result["candidates"] + + +def test_column_permissions_also_bound_relation_context(table_environment): + from sqlbot_xpack.permissions.models.ds_permission import DsPermission + + from apps.system.models.system_model import UserWsModel + + database, client, ds_id = table_environment + configure(client, table_count=1) + for model in (DsPermission, UserWsModel): + model.__table__.create(database) + with Session(database) as session: + session.add(UserWsModel(uid=2, oid=1, weight=0)) + beta = ( + session.query(CoreTable).filter(CoreTable.table_name == "vector_beta").one() + ) + alpha = ( + session.query(CoreTable).filter(CoreTable.table_name == "sales_alpha").one() + ) + alpha_field = ( + session.query(CoreField).filter(CoreField.table_id == alpha.id).one() + ) + hidden = CoreField( + ds_id=ds_id, + table_id=beta.id, + field_name="secret_join", + field_type="INTEGER", + field_index=2, + ) + session.add(hidden) + session.flush() + permission = DsPermission( + name="hide secret", + type="column", + ds_id=ds_id, + table_id=beta.id, + permissions=json.dumps([{"field_id": hidden.id, "enable": False}]), + enable=True, + ) + session.add(permission) + session.flush() + session.add( + DsRules( + oid=1, + name="restricted", + permission_list=[permission.id], + user_list=[2], + enable=True, + ) + ) + ds = session.get(CoreDatasource, ds_id) + ds.table_relation = [ + { + "shape": "edge", + "source": {"cell": str(beta.id), "port": str(hidden.id)}, + "target": {"cell": str(alpha.id), "port": str(alpha_field.id)}, + } + ] + session.add(ds) + session.commit() + schema = run(task(database, ds_id, user_id=2)) + assert "vector_beta" in schema and "secret_join" not in schema + assert "sales_alpha" not in schema and "Foreign keys" not in schema + + +def test_sample_data_uses_the_selected_tables(table_environment, monkeypatch): + database, client, ds_id = table_environment + configure(client, table_count=1) + monkeypatch.setattr(settings, "SAMPLE_DATA_ENABLED", True) + # Substitute the external datasource query, keeping sample filtering/formatting real. + monkeypatch.setattr( + "apps.datasource.crud.datasource.exec_sql", lambda **_kw: {"data": [{"id": 42}]} + ) + current = task(database, ds_id) + run(current) + assert "# Table: vector_beta" in current.chat_question.sample_data + assert '"id": 42' in current.chat_question.sample_data + assert ( + "sales_alpha" not in current.chat_question.sample_data + and "hidden_table" not in current.chat_question.sample_data + ) + assert any( + current.chat_question.sample_data in context for context in current.llm.contexts + ) + + +def test_datasource_answer_persistence_uses_original_task_switch( + table_environment, monkeypatch +): + database, client, ds_id = table_environment + add_datasource_candidates(database) + monkeypatch.setattr(settings, "TABLE_EMBEDDING_ENABLED", False) + configure(client, table_count=1, datasource_count=1, keyword_enabled=False) + current = task(database, None) + current.llm = QuestionProvider( + datasource={"id": ds_id, "name": "accessible"}, + on_datasource=lambda: monkeypatch.setattr( + settings, "TABLE_EMBEDDING_ENABLED", True + ), + ) + schema = run(current) + assert "second_candidate" in current.llm.contexts[0] + assert all(name in schema for name in ("sales_alpha", "vector_beta", "other_gamma")) + with Session(database) as session: + answer = session.get(ChatRecord, current.record.id).datasource_select_answer + assert json.loads(json.loads(answer)["content"])["id"] == ds_id diff --git a/docs/adr/0001-runtime-embedding-configuration.md b/docs/adr/0001-runtime-embedding-configuration.md new file mode 100644 index 00000000..f8e75dfc --- /dev/null +++ b/docs/adr/0001-runtime-embedding-configuration.md @@ -0,0 +1,15 @@ +# 向量召回策略采用系统级运行时配置 + +系统管理员在参数配置页面统一管理知识召回、表筛选和数据源预筛选的策略参数。向量模型及模型目录继续作为部署配置，不在本次引入模型切换；模型切换涉及已有向量兼容与重建，不能按普通召回参数处理。配置适用于所有工作空间，各入口仍遵循自身的功能和知识作用域。 + +有效值按“已保存的界面值 > 启动配置 > 代码默认值”解析，页面显示后端解析出的有效值。未保存过的参数沿用已有部署配置，保存后由持久化值覆盖。术语和 SQL 示例分别配置相似度与候选数量，不增加一层公共默认值继承。 + +配置保存后无需重启，从各服务进程的下一次问数或后台任务开始生效；任务开始时取得配置快照，执行期间保持一致。关闭开关允许已开始的向量生成任务完成，避免一次问数的筛选规则中途变化。 + +开关同时控制向量生成与向量召回。关闭期间不开始新的向量生成，但内容修改必须使旧向量失效；重新开启后异步补齐缺失或失效向量，保留未失效的向量。仅在开启时扫描空向量无法覆盖关闭期间修改过的已有数据。 + +向量补齐期间保持问数可用：知识召回保留字面匹配及已有有效向量；候选表或数据源的向量不完整时，跳过该层向量预筛选，保留原本可访问的候选集合。接受暂时增加上下文与耗时，以避免不完整向量提前排除候选。 + +保留现有开关关系和召回语义。新的页面保存严格校验相似度和权重为 `[0,1]` 内有限数值、数量为正整数，非法输入整次拒绝保存；不顺带改变既有启动配置解析行为。 + +本决策已由 `common/core/embedding_config.py` 的统一类型与直接持久化读取、`LLMService` 任务快照，以及 `embedding_lifecycle.py` / 数据源任务的资源锁和条件写回实现。保存使用参数写入事务锁，后台任务在工作线程开始时读取有效配置；不依赖 xpack 的进程内参数缓存传播。实现状态不等同于所有部署环境均已验收；完整参数、调用链、可重复验证与剩余范围见[设计与影响范围](../design/embedding-parameter-configuration.md)。 diff --git a/docs/agents/backend.md b/docs/agents/backend.md index 5bb29664..6dca5312 100644 --- a/docs/agents/backend.md +++ b/docs/agents/backend.md @@ -54,6 +54,8 @@ Endpoint 命名沿既有风格： - 创建/更新前完成重复性、必填和作用域校验；不要先写入再依赖唯一约束报错。 - 事务提交通常发生在 CRUD/service 边界；API handler 不要随意中途 commit。 - embedding、缓存清理或后台任务沿用当前领域的线程任务模式，不在请求路径中阻塞等待长任务。 +- 知识向量任务通过 `common/utils/embedding_lifecycle.py` 在实际工作线程开始时读取持久化配置；术语主词/同义词与 SQL 示例问题变化必须在业务事务内使受影响向量失效。生成任务使用按资源的 PostgreSQL 事务 advisory lock、锁后重读和输入条件写回，不能用进程内锁或无条件写回替代。失败向量保持空值，有限重试失败后报告异常，后续资源任务、重新开启和进程启动均可重试。 +- 表与数据源向量任务入口在 `apps/datasource/crud/table.py`，复用上述资源锁并兼容 JSON 向量。元数据写入先按父数据源、子表顺序在同一事务内使向量失效（`embedding_state.py`）；生成期间不持有业务行锁，发布前按相同顺序锁行、重读并比较完整输入，防止旧结果覆盖修改。启动与重新开启扫描修复缺失或无效向量，保留有效向量；启动补齐不使用进程生命周期级单次锁。 ## 异步与阻塞 diff --git a/docs/agents/domain.md b/docs/agents/domain.md new file mode 100644 index 00000000..d818d1a4 --- /dev/null +++ b/docs/agents/domain.md @@ -0,0 +1,22 @@ +# Domain Docs + +本仓库采用 single-context 布局。 + +## 阅读入口 + +探索相关领域前： + +- 读取根目录 `CONTEXT.md`：它是本仓库的领域词汇表， + 对应工程技能默认称作 GLOSSARY.md 的文档。 +- 若 `docs/adr/` 存在，读取与当前任务相关的架构决策。 +- 领域边界尚不明确时，按 `AGENTS.md` 查阅 + `docs/agents/domain-open-questions.md`。 + +## 维护约定 + +- 技能要求读取或更新 GLOSSARY.md 时，操作 `CONTEXT.md`。 +- Issue、设计建议和测试命名使用 `CONTEXT.md` 中定义的术语。 +- 术语缺口先核实；稳定定义补入 `CONTEXT.md`。 +- ADR 在实际形成架构决策时按需创建，保存为 + `docs/adr/NNNN-.md`；缺少目录时继续当前任务。 +- 建议与既有 ADR 冲突时，明确指出冲突及重新讨论的理由。 diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md new file mode 100644 index 00000000..dd2c78a9 --- /dev/null +++ b/docs/agents/issue-tracker.md @@ -0,0 +1,39 @@ +# Issue tracker: GitHub + +本仓库的 Issue 和规格维护在 dataease/SQLBot 的 GitHub Issues。 +使用 gh CLI，并显式指定 `--repo dataease/SQLBot`。 + +## 操作约定 + +- 创建：`gh issue create --repo dataease/SQLBot --title "..." --body-file ` +- 读取：`gh issue view --repo dataease/SQLBot --json number,title,body,labels,comments` +- 列表：`gh issue list --repo dataease/SQLBot --state open --json number,title,body,labels` +- 评论：`gh issue comment --repo dataease/SQLBot --body-file ` +- 标签：`gh issue edit --repo dataease/SQLBot --add-label "..."` + 或 `--remove-label "..."` +- 关闭：`gh issue close --repo dataease/SQLBot` + +多行正文先写入临时文件，再通过 `--body-file` 传入。 +标签角色映射见 `triage-labels.md`。 + +## Pull requests as a triage surface + +**PRs as a request surface: no.** + +## 技能用语 + +- “publish to the issue tracker”：创建 GitHub Issue。 +- “fetch the relevant ticket”：读取对应 Issue 及评论。 + +## Wayfinding + +地图使用一个带 `wayfinder:map` 标签的 Issue；子任务各建一个 Issue， +使用 `wayfinder:` 标签，type 为 research、prototype、grilling 或 task。 + +父子关系和阻塞关系优先使用 GitHub 原生功能；不可用时， +在地图正文维护任务列表，在子任务顶部写 `Part of #` 和 +`Blocked by: #`。 + +按地图顺序选择未关闭、无未完成阻塞项且无人认领的任务。 +开始前分配给当前开发者；完成后评论结果、关闭任务， +并将结论和链接更新到地图的 Decisions-so-far。 diff --git a/docs/agents/knowledge-enhancement.md b/docs/agents/knowledge-enhancement.md index da1ec7ae..3e4df0f7 100644 --- a/docs/agents/knowledge-enhancement.md +++ b/docs/agents/knowledge-enhancement.md @@ -25,11 +25,21 @@ - SQL 示例：用完整问题做双向包含匹配（问题包含示例问题，或示例问题包含问题）。 - 自定义提示词：不做问题匹配，作用域内全部注入。 - 关键词/包含匹配与 embedding 召回是并集关系，不是优先级关系：embedding 只是补充召回渠道，不淘汰关键词命中的结果。 -- `EMBEDDING_ENABLED=false` 时退化为纯关键词/包含匹配。 -- embedding 召回按相似度降序取 top N（默认 5）、相似度阈值默认 0.4，由 `EMBEDDING_*_SIMILARITY` / `EMBEDDING_*_TOP_COUNT` 配置。 +- 有效知识向量开关 `embedding.enabled=false` 时退化为纯关键词/包含匹配；它不控制表与数据源向量。 +- embedding 召回按相似度降序取 top N（代码默认 5），相似度严格大于阈值（代码默认 0.4）。术语和 SQL 示例分别使用 `embedding.terminology_*` / `embedding.data_training_*` 的有效参数；多关键词和字面匹配合并后可超过 top N。 - 注入顺序固定：自定义提示词 → 术语 → SQL 示例，各自独立成段附在 schema 信息之后。 - 术语另有一个独立用途：提取的关键词经术语同义词扩展后用于表匹配，与术语注入互不影响。 +## 运行时配置与向量生命周期 + +系统管理员在参数配置页保存系统级 `embedding.*` 参数；有效值按“持久化值 > 对应启动配置 > 代码默认值”解析。未保存的参数沿用 `EMBEDDING_*` / `TABLE_EMBEDDING_*` / `DS_EMBEDDING_COUNT` 部署值；保存后仅改变环境变量不会覆盖持久化值。页面 GET 显示同一有效值，保存前整批校验类型和范围。定义与读取入口为 `common/core/embedding_config.py`，不修改全局 `settings` 或依赖进程内参数缓存。 + +`LLMService` 创建任务时读取不可变快照，后续知识召回、数据源和表筛选共用该快照；新发起分析重新读取配置并保留记录中的历史关键词。后台向量任务在实际工作线程开始时读取配置，排队时间不算任务开始。保存后的新任务读取数据库提交，已开始任务可按原快照完成。 + +关闭期间不开始对应向量生成，但业务事务仍使输入变化的向量失效；重新开启和进程启动异步补齐，保留有效向量。术语主词/同义词、SQL 示例问题改变时清空对应向量；表字段变化沿表和父数据源传播失效。资源任务通过 PostgreSQL advisory lock 去重、锁后重读并按输入条件发布，防止旧结果覆盖已编辑或删除资源。失败有限重试后保持待补齐并报告异常，后续资源任务、重新开启或启动可重试。入口见 `common/utils/embedding_lifecycle.py`、`common/utils/embedding_threads.py` 和 `apps/datasource/crud/embedding_state.py`。 + +表与数据源使用独立的 `embedding.table_enabled` 开关；候选向量不完整或评分失败时，跳过该层向量预筛选，保留原本可访问候选。关键词提取与同义词扩展开关 `embedding.keyword_enabled` 独立于两个向量开关。完整参数与边界见 [设计与验证说明](../design/embedding-parameter-configuration.md)。 + ## 已知边界 - 关键词/包含匹配的命中数量没有上限，极端情况下可能注入大量内容（token 风险）；只有 embedding 召回受 top N 限制。 diff --git a/docs/agents/triage-labels.md b/docs/agents/triage-labels.md new file mode 100644 index 00000000..76e1e4c4 --- /dev/null +++ b/docs/agents/triage-labels.md @@ -0,0 +1,14 @@ +# Triage Labels + +工程技能的五个角色映射如下： + +| 技能角色 | GitHub 标签 | 含义 | +| --- | --- | --- | +| needs-triage | needs-triage | 等待维护者评估 | +| needs-info | needs-info | 等待报告者补充信息 | +| ready-for-agent | ready-for-agent | 规格完整，可交给 Agent 实现 | +| ready-for-human | ready-for-human | 需要人工实现 | +| wontfix | wontfix | 不予处理 | + +技能提及某个角色时，使用本表对应的 GitHub 标签。 +已有业务标签继续保留。 diff --git a/docs/design/embedding-parameter-configuration.md b/docs/design/embedding-parameter-configuration.md new file mode 100644 index 00000000..4ad3ec6a --- /dev/null +++ b/docs/design/embedding-parameter-configuration.md @@ -0,0 +1,138 @@ +# 向量化参数界面配置设计与影响范围 + +将 10 个向量召回策略参数从仅启动时配置扩展为系统管理员可在 `system/setting/parameter` 页面配置。系统级参数覆盖部署默认值，并从下一次问数或后台任务开始生效。模型及目录仍是启动配置。核心决策见 [ADR 0001](../adr/0001-runtime-embedding-configuration.md)。 + +正式规格已发布为 [GitHub Issue #1403](https://github.com/dataease/SQLBot/issues/1403)，包含完整用户故事、实现决策和已确认的测试边界；本文保留源码影响索引。 + +本文已随 #1403 实现更新；下列源码链接是运行时行为的维护入口，验收条件仍用于区分实现与验证范围。持久化使用已发布 xpack wheel，未修改闭源源码。术语沿用根目录 [CONTEXT.md](../../CONTEXT.md)。 + +## 参数范围 + +启动默认值定义位于 [config.py](../../backend/common/core/config.py) 的 `Settings`，持久化键与不可变配置模型见 [embedding_config.py](../../backend/common/core/embedding_config.py)。 + +| 启动配置名 | 代码默认值 | 页面含义与实际影响 | +| --- | --- | --- | +| `EMBEDDING_ENABLED` | `true` | 知识向量化：共同控制术语与 SQL 示例的向量生成、补齐和召回，不是全系统总开关 | +| `EMBEDDING_TERMINOLOGY_SIMILARITY` | `0.4` | 术语向量召回相似度阈值，现状为严格大于阈值 | +| `EMBEDDING_TERMINOLOGY_TOP_COUNT` | `5` | 每个关键词的术语向量候选数量；多个关键词和字面匹配合并后可超出该数量 | +| `EMBEDDING_DATA_TRAINING_SIMILARITY` | `0.4` | SQL 示例向量召回相似度阈值，现状为严格大于阈值 | +| `EMBEDDING_DATA_TRAINING_TOP_COUNT` | `5` | SQL 示例向量候选数量；合并字面匹配后可超出该数量 | +| `TABLE_EMBEDDING_ENABLED` | `true` | 共同控制表与数据源向量生成、补齐和预筛选 | +| `TABLE_EMBEDDING_COUNT` | `10` | 表召回候选数量；明确提及的表、关系补全表可以超出该数量 | +| `TABLE_EMBEDDING_ALPHA` | `0.4` | 表混合匹配的向量权重，关键词权重为 `1 - alpha` | +| `TABLE_EMBEDDING_KEYWORD_ENABLED` | `true` | 关键词提取、术语同义词扩展与表混合匹配；独立于表向量开关，也影响术语匹配输入 | +| `DS_EMBEDDING_COUNT` | `10` | 自动选择数据源之前的向量预筛选候选数量 | + +以下配置不增加页面控件： + +- `DEFAULT_EMBEDDING_MODEL`、`LOCAL_MODEL_PATH`：保留部署配置。现有模型名称仅用于缓存默认 key，实际加载目录仍硬编码模型子目录，不具备通用模型切换能力。 +- `EMBEDDING_DEFAULT_SIMILARITY`、`EMBEDDING_DEFAULT_TOP_COUNT`：仅在 `Settings` 类定义时提供具体字段的初始默认值，业务没有独立消费者。现有环境变量修改这两个字段并不会自动更新具体字段，不引入新的公共默认值继承。 +- `SAMPLE_DATA_ENABLED`：属于表样例数据控制，本次不迁移。 + +## 已实现行为 + +系统管理员统一配置，所有工作空间共享。适用于页面、MCP、普通小助手、页面嵌入助手及高级应用各自已经使用这些参数的功能，不扩大高级应用的表或数据源筛选能力。 + +有效值优先级为已保存的页面值、启动配置、代码默认值。页面读取后端计算出的有效值，避免前端硬编码默认值覆盖既有部署配置。保存后，后续环境变量变化不覆盖该持久化值。 + +每次问数或后台任务在开始时读取一份一致的配置，执行中不切换规则。保存成功后的新任务无需重启即可使用新值，各服务进程必须一致。已开始的向量生成任务允许完成。 + +关闭知识或表向量开关后，不再开始对应向量生成或召回；修改内容时使受影响的旧向量失效。重新开启后异步补齐缺失或失效向量，保留仍有效的向量。旧任务不得将过期内容的向量写回；内容变更和向量写回之间必须有一致性保护。 + +补齐中或向量生成失败时，术语和 SQL 示例继续字面匹配，并使用已有有效向量补充召回。当前候选表或数据源存在缺失、失效向量时，跳过该层向量预筛选，交由后续流程处理原本可访问的候选集合；模型调用失败也应保留可用的降级路径。该行为可能增加上下文和耗时，不保证外部模型能够处理无限大的候选集合。 + +保留现有语义：知识向量召回与字面匹配取并集；表和数据源共享开关；关键词开关独立；历史记录保存的关键词仍可供后续分析使用。页面按知识召回、表与数据源筛选分组，不能因关闭表向量开关而把仍生效的关键词开关隐藏或视为无效。 + +页面与后端均校验：相似度和权重是 `[0,1]` 内的有限数值，候选数量是正整数，不新增任意数量上限。非法输入整次拒绝保存，不静默截断、替换默认值或部分写入。该规则针对新页面保存，不顺带重定义既有启动配置解析行为。 + +## 参数配置调用链 + +| 环节 | 源码入口 | 现状与改动关注点 | +| --- | --- | --- | +| 路由与页面 | [router/index.ts](../../frontend/src/router/index.ts)；[parameter/index.vue](../../frontend/src/views/system/parameter/index.vue) | `setting/parameter`；现有 `chat.limit_rows` 为参考。旧 `chat/login/platform` 控件保留；十个 `embedding.*` 字段按明确键与类型独立绑定、保存和回显 | +| 值解析 | [utils.ts](../../frontend/src/utils/utils.ts) 的 `formatArg` | 旧配置沿用 `formatArg`；新字段按布尔、数量和小数解析，避免把 `0/1` 混成布尔值 | +| 参数 API | [parameter.py](../../backend/apps/system/api/parameter.py) 的 `get_args/save_args` | GET/POST `/system/parameter` 要求系统管理员权限；保存有参数设置审计 | +| 持久化适配 | [parameter_manage.py](../../backend/apps/system/crud/parameter_manage.py) | 保存委托已发布 xpack 的 `save_group_args`；写入前整批校验并串行化参数写事务；有效向量配置直接读取 `sys_arg`，不依赖适配器缓存 | +| 存储结构 | [035_sys_arg_ddl.py](../../backend/alembic/versions/035_sys_arg_ddl.py) | `sys_arg` 使用 `pkey/pval`，没有工作空间维度；不因本次配置迁移引入工作空间覆盖 | +| 任务读取 | [llm.py](../../backend/apps/chat/task/llm.py) 的 `LLMService.create` | 构造任务时加载不可变 `EmbeddingConfig`；旧 `chat.*` 参数仍在 `create` 中加载，不修改全局 `settings` | + +已用发布 wheel 验证未知 `embedding.*` 键的保存与读取。运行时读取不依赖适配器的缓存失效：每次任务以单条数据库查询取得全部覆盖值。历史迁移缺少 `pkey` 唯一约束，参数写事务使用 advisory lock 串行化；历史重复键按最大 ID 确定有效值，保存可更新该值。本次不新增迁移或修改 wheel。 + +## 向量生成与数据变更链 + +| 功能 | 源码入口与函数 | 影响 | +| --- | --- | --- | +| 模型加载 | [ai_model/embedding.py](../../backend/apps/ai_model/embedding.py) 的 `local_embedding_model`、`EmbeddingModelCache` | 模型路径及默认参数在导入时确定，模型对象为进程内缓存；本次不切模型 | +| 启动补齐 | [main.py](../../backend/main.py) 的 `lifespan`、三个 `init_*embedding*` 函数 | 迁移和缓存初始化后提交补齐；各进程均可恢复待补齐任务，工作线程读取有效开关 | +| 后台任务提交 | [embedding_threads.py](../../backend/common/utils/embedding_threads.py) 的全部 `run_save_*`、`fill_empty_*` | 术语、SQL 示例、表、数据源共用线程池和独立 session；任务执行时取得配置，不能遗漏这一读取路径 | +| 启动任务互斥 | [distributed_lock.py](../../backend/common/utils/distributed_lock.py) 的 `SingleWorkerGuard` | 旧启动锁不再承担向量补齐去重；`embedding_lifecycle.py` 按资源事务锁、锁后重读和条件发布协调各进程 | +| 术语新增、编辑、导入 | [terminology.py](../../backend/apps/terminology/curd/terminology.py) 的 `create_terminology/update_terminology/batch_create_terminology` | 提交父术语 id，生成覆盖主词和同义词；输入改变与向量失效在同一业务事务内 | +| 术语生成与补齐 | 同文件 `save_embeddings/run_fill_empty_embeddings` | 任务开始读取知识开关；业务修改先将过期向量清空，补齐只生成空向量并保留有效值 | +| SQL 示例新增、编辑、导入 | [data_training.py](../../backend/apps/data_training/curd/data_training.py) 的 `create_training/update_training/batch_create_training` | 向量输入为示例问题；问题变更时原子清空旧向量 | +| SQL 示例生成与补齐 | 同文件 `save_embeddings/run_fill_empty_embeddings` | 任务开始读取知识开关，补齐空向量；按问题输入条件写回 | +| 数据源创建、选择表与同步 | [datasource.py](../../backend/apps/datasource/crud/datasource.py) 的 `create_ds/chooseTables/sync_table/sync_single_fields` | 创建及选择表进入同步链；同步更新表和所属数据源向量 | +| 数据源与元数据编辑 | 同文件 `update_ds/update_table_and_fields/updateTable/updateField` | 名称、描述、表和字段内容影响表或数据源向量；失效必须沿依赖传播 | +| 表与数据源生成和补齐 | [table.py](../../backend/apps/datasource/crud/table.py) 的 `save_table_embedding/save_ds_embedding/run_fill_empty_table_and_ds_embedding` | 共用表向量开关；数据源向量包含名称、描述及表字段结构；扫描空值和无效 JSON 向量，保留有效向量；发布前重读完整输入 | +| 向量存储 | [terminology_model.py](../../backend/apps/terminology/models/terminology_model.py)、[data_training_model.py](../../backend/apps/data_training/models/data_training_model.py)、[datasource/models/datasource.py](../../backend/apps/datasource/models/datasource.py) | 术语和 SQL 示例使用 pgvector；表和数据源使用 Text JSON，失效/补齐需覆盖两种表示 | + +## 问数及其他消费链 + +| 功能 | 源码入口与函数 | 影响与边界 | +| --- | --- | --- | +| 术语召回 | [terminology.py](../../backend/apps/terminology/curd/terminology.py) 的 `select_terminology_by_word/get_terminology_template` | 工作空间、数据源、高级应用三条向量 SQL；与字面匹配合并，按关键词逐次召回 | +| SQL 示例召回 | [data_training.py](../../backend/apps/data_training/curd/data_training.py) 的 `select_training_by_question/get_training_template` | 数据源和高级应用两条向量 SQL；与双向包含匹配合并 | +| SQL 上下文注入 | [llm.py](../../backend/apps/chat/task/llm.py) 的 `filter_terminology_template/filter_training_template/init_messages` | 自动选数据源和已指定数据源两条问数路径都需传递同一配置快照；保持知识作用域隔离 | +| 自动选择数据源 | 同文件 `select_datasource`；[ds_embedding.py](../../backend/apps/datasource/embedding/ds_embedding.py) 的 `get_ds_embedding` | 多数据源时先截断向量候选，再交 LLM 选择；表开关还影响选择结果持久化分支，不可遗漏 | +| 表结构筛选 | `llm.py` 的 `choose_table_schema`；[datasource.py](../../backend/apps/datasource/crud/datasource.py) 的 `get_table_schema/get_table_obj_by_ds` | 使用已有权限及选中状态约束后的候选；开关关闭保留该候选集合；配置不能绕过权限过滤 | +| 表混合匹配 | [table_embedding.py](../../backend/apps/datasource/embedding/table_embedding.py) 的 `calc_table_embedding/_calc_vector_only/_select_tables` | 关键词与向量按 alpha 融合，保留问题明确提到的表，再按数量补充；异常或不完整向量的降级不能任意丢表 | +| 关系与样例数据 | `datasource.py` 的 `get_table_schema/get_tables_sample_data`；`llm.py` 的 `choose_table_schema` | 筛选后仅从原授权表和字段补充关系上下文，明确表与关系表可超出候选数；样例数据使用最终选表 | +| 关键词提取和同义词扩展 | `llm.py` 的 `extract_keywords` 及两条问数入口；`terminology.py` 的 `expand_with_terminology` | 关键词开关控制额外 LLM 调用、术语匹配输入及扩展结果保存，即使表向量关闭仍可生效 | +| 分析 | `llm.py` 的 `generate_analysis` | 使用记录已保存的关键词并重新检索术语，后续新发起分析受新知识召回参数影响 | +| 预测 | 同文件 `generate_predict` | 当前不重新筛选术语，不能声称直接采用新术语阈值；可间接使用之前形成的上下文 | +| 推荐问题 | 同文件推荐问题生成链 | 独立获取 schema 时显式 `embedding=False`；问数链内可能复用此前已筛选 schema | +| 已选表上下文重建 | 同文件 `run_task` 中生成图表前调用 `get_table_schema` 的路径 | 显式 `embedding=False`，不对已确定的表再次筛选 | +| 页面、普通及嵌入助手 | [chat/api/chat.py](../../backend/apps/chat/api/chat.py) 的 `question_answer_inner` 等任务创建入口 | 进入共同 `LLMService` 流程；各任务都需取得有效配置 | +| MCP | [mcp.py](../../backend/apps/mcp/mcp.py) 的问数入口 | 也调用 `question_answer_inner`；现有 `embedding` 入参没有被 `LLMService` 消费，不能据此判断 MCP 不使用向量功能 | +| 高级应用 | [assistant.py](../../backend/apps/system/crud/assistant.py) 的 `AssistantOutDs.get_db_schema`；`llm.py` 的助手分支 | 术语与 SQL 示例仍按高级应用作用域召回；表向量调用被注释，数据源向量预筛选被当前调用方排除，本次不新增 | + +`table_embedding.py` 的旧 `get_table_embedding` 仍读取表数量，但当前没有活动调用。`ds_embedding.py` 内虽保留高级应用动态向量分支，当前唯一调用方却排除了该类助手。维护时应区分活动链路与这些遗留分支，不把后者描述为已有产品能力。 + +## 实现约束与验收条件 + +五条术语和 SQL 示例向量 SQL 已改为执行时参数绑定阈值和数量。统一不可变 `EmbeddingConfig` 及持久化读取负责优先级，问数快照和后台任务共用该入口。知识生成使用资源事务锁与输入条件写回；表和数据源发布前按父数据源、子表顺序锁行、重读并比较完整输入。失败有限重试后保持未完成状态，后续任务或进程启动可恢复。 + +| 验收场景 | 可观察结果 | +| --- | --- | +| 首次打开与已有部署配置 | 10 个参数显示后端有效值；未保存值沿用启动配置，刷新不改变 | +| 保存与再次打开 | 布尔、整数、小数正确回显；持久化值优先于启动配置；其他已有参数保留 | +| 非法输入 | 越界、非有限小数、非整数或非正数量被前后端拒绝；整次保存无部分更新 | +| 权限 | 系统管理员可以读取和保存；非管理员拒绝保存且无持久化变化 | +| 热更新与任务一致性 | 保存后所有 worker 与 MCP 新任务使用新值；在途任务继续使用原配置 | +| 知识召回 | 关闭后保留字面匹配；阈值和数量在新查询生效；覆盖术语三个作用域和 SQL 示例两个作用域 | +| 表与数据源筛选 | 数量、alpha、关键词开关在新任务生效；明确表和关系补全保留；权限边界不扩大 | +| 关闭期间新增及修改 | 不启动对应向量生成；新增向量为空，修改过的旧向量不可继续视为有效 | +| 重新开启与重启 | 自动补齐缺失/失效向量；并发保存、多个进程与重启不导致漏补或重复失控 | +| 内容修改与旧任务完成竞态 | 旧内容的生成结果不会覆盖新内容的失效状态或新向量 | +| 补齐中与失败 | 知识匹配仍可用；表/数据源向量不完整时不因预筛选漏候选；失败不伪报完成 | +| 不同入口与后续任务 | 覆盖页面、MCP、普通/嵌入助手、高级应用知识作用域及分析重新检索；未使用向量的路径保持边界 | +| 页面说明 | 两组参数、开关关系和候选数量含义准确；关键词开关保持独立可理解 | + +[test_table_embedding.py](../../backend/tests/test_table_embedding.py) 覆盖表匹配逻辑；参数与生命周期测试进一步覆盖配置解析、持久化覆盖、任务快照、失效与补齐和并发写回。真实 API/数据库、独立 MCP 传输和页面保存回显分别记录证据；构建和源码守卫不能替代这些验证。 + +规格确认采用两个既有业务测试边界：参数管理的真实读取/保存接口，以及问数、分析和资源变更的任务入口。前者观察权限、有效值和持久化结果，后者观察候选、模型上下文和向量结果；使用可控模型提供者复现边界与并发，并另行以真实数据库、多进程和浏览器验证集成行为。 + +## 验证入口与限制 + +从 `backend/` 执行默认离线回归；集成测试必须显式选择授权的可销毁 PostgreSQL/pgvector 数据库，每个用例创建并清理独立 schema： + +```bash +uv run --no-sync pytest -q +SQLBOT_RUN_EMBEDDING_INTEGRATION=1 SQLBOT_TEST_DATABASE_URL="$DISPOSABLE_DATABASE_URL" \ + uv run --no-sync pytest -q tests/test_embedding_parameters_integration.py \ + tests/test_knowledge_lifecycle_integration.py tests/test_table_parameters_integration.py \ + tests/test_datasource_lifecycle_integration.py tests/test_mcp_parameters_integration.py +``` + +前端类型与打包在 `frontend/` 执行 `npm run build`。使用已准备环境时保留 `--no-sync`；未入库 lockfile，源码 SHA 本身不能冻结依赖。验证采用发布 wheel `sqlbot-xpack==0.0.5.36`、PostgreSQL 17/pgvector 和受控模型提供者；未调用外部 LLM，未验证模型语义质量、真实外部数据源、生产多机部署或镜像安装。参数快照、资源并发和锁释放由真实 PostgreSQL、多进程测试验证，不能用直接构造 `LLMService` 的用例代替 MCP 网络入口验收。 + +MCP 集成用例通过独立 Uvicorn `main:mcp_app` 的 SSE 传输调用 `mcp_start/mcp_question`，另一个 API 进程保存参数；仅控制模型和外部数据源提供者。fixture 建立 schema，因此 API 进程不运行迁移 lifespan；MCP 进程保留正常 lifespan。该用例不覆盖完整部署启动或图表图片服务。浏览器逐控件保存与刷新结果单独记录，不以构建或 API 测试代替。 diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 48fc79e4..e0264ad5 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -69,6 +69,20 @@ "return_to_view": "Return to View" }, "parameter": { + "embedding_scope_hint": "These settings apply to all workspaces and take effect for new questions and background tasks after saving.", + "embedding_backfill_hint": "Enabling generates and uses vectors, with missing or stale vectors filled in the background. Disabling stops new generation; tasks already started may finish.", + "terminology": "Terminology", + "sql_example": "SQL examples", + "knowledge_type": "Knowledge type", + "similarity_threshold": "Similarity threshold", + "vector_candidates": "Vector candidates", + "similarity_hint": "Range: 0–1. Higher values require closer matches.", + "terminology_count_hint": "Candidates per keyword vector search. Combining multiple keywords and text matches may exceed this count.", + "sql_example_count_hint": "Candidates per vector search. Combining text matches may exceed this count.", + "table_count_hint": "Candidates retained by vector filtering. Explicitly mentioned tables and related tables may increase the final count.", + "datasource_count_hint": "Datasource candidates retained by vector filtering before automatic datasource selection.", + "table_alpha_hint": "Range: 0–1. Higher values favor vector matching; keyword weight is 1 minus this value. Used for hybrid scoring only when both table vector filtering and keyword extraction are enabled.", + "table_alpha_disabled_hint": "Enable keyword extraction to adjust the vector weight for hybrid matching.", "execution_details": "Execution Details", "overview": "Overview", "tokens_required": "Tokens Required", @@ -98,7 +112,26 @@ "login_settings": "Login Settings", "default_login": "Default Login Method", "disable_initial_password": "Disable Initial Password", - "hide_initial_password": "Hide Initial Password on Page" + "hide_initial_password": "Hide Initial Password on Page", + "knowledge_enabled": "Vector matching", + "knowledge_enabled_hint": "Match additional terminology and SQL examples by semantic similarity. Text matching remains available when disabled.", + "table_enabled": "Vector filtering", + "table_enabled_hint": "Filter relevant tables and datasources for each question. When disabled, retain all accessible candidates.", + "table_selection": "Table and datasource selection", + "table_count": "Table candidates", + "datasource_count": "Datasource candidates", + "table_alpha": "Vector weight for table matching", + "keyword_enabled": "Keyword extraction", + "table_candidates_hint": "Table candidate count is not a final table limit: explicitly mentioned tables and related tables are retained. Datasource count limits vector candidates before the model selects a datasource.", + "table_weight_hint": "Vector weight ranges from 0 to 1; keyword weight is 1 minus this value.", + "keyword_enabled_hint": "Extract keywords from questions for terminology and table matching. Works independently of the vector switches.", + "knowledge_retrieval": "Knowledge matching", + "terminology_similarity": "Terminology similarity threshold", + "terminology_top_count": "Terminology vector candidates per keyword", + "data_training_similarity": "SQL example similarity threshold", + "data_training_top_count": "SQL example vector candidates", + "knowledge_candidates_hint": "Candidate counts apply to vector recall. Literal matches and multiple keywords can increase the final injected knowledge count. Saved settings apply to new tasks.", + "invalid_embedding_parameter": "Similarity must be a finite number from 0 to 1; candidate counts must be positive integers." }, "prompt": { "default_password": "Default password:{msg}", diff --git a/frontend/src/i18n/ko-KR.json b/frontend/src/i18n/ko-KR.json index 21097278..6d57c8f6 100644 --- a/frontend/src/i18n/ko-KR.json +++ b/frontend/src/i18n/ko-KR.json @@ -69,6 +69,20 @@ "return_to_view": "화면으로 돌아가기" }, "parameter": { + "embedding_scope_hint": "이 설정은 모든 워크스페이스에 적용되며, 저장 후 시작하는 질문과 백그라운드 작업부터 적용됩니다.", + "embedding_backfill_hint": "활성화하면 벡터를 생성하고 사용하며, 누락되거나 오래된 벡터는 백그라운드에서 보충합니다. 비활성화하면 새 벡터 생성을 중지하지만 이미 시작된 작업은 완료될 수 있습니다.", + "terminology": "용어", + "sql_example": "SQL 예제", + "knowledge_type": "지식 유형", + "similarity_threshold": "유사도 임계값", + "vector_candidates": "벡터 후보 수", + "similarity_hint": "범위: 0~1. 값이 높을수록 더 엄격하게 매칭합니다.", + "terminology_count_hint": "키워드별 벡터 검색 후보 수입니다. 여러 키워드와 텍스트 매칭을 합치면 최종 개수가 이 값을 초과할 수 있습니다.", + "sql_example_count_hint": "벡터 검색 후보 수입니다. 텍스트 매칭을 합치면 최종 개수가 이 값을 초과할 수 있습니다.", + "table_count_hint": "벡터 필터링으로 유지할 후보 수입니다. 명시적으로 언급된 테이블과 관계 보완 테이블로 인해 최종 개수가 늘어날 수 있습니다.", + "datasource_count_hint": "자동 데이터 소스 선택 전에 벡터 필터링으로 유지할 데이터 소스 후보 수입니다.", + "table_alpha_hint": "범위: 0~1. 값이 높을수록 벡터 매칭을 우선하며, 키워드 가중치는 1에서 이 값을 뺀 값입니다. 테이블 벡터 필터링과 키워드 추출이 모두 활성화된 경우에만 혼합 점수에 사용됩니다.", + "table_alpha_disabled_hint": "키워드 추출을 활성화하면 혼합 매칭의 벡터 가중치를 조정할 수 있습니다.", "execution_details": "실행 세부 정보", "overview": "개요", "tokens_required": "필요한 토큰 수", @@ -98,7 +112,26 @@ "login_settings": "로그인 설정", "default_login": "기본 로그인 방식", "disable_initial_password": "초기 비밀번호 비활성화", - "hide_initial_password": "페이지에서 초기 비밀번호 숨기기" + "hide_initial_password": "페이지에서 초기 비밀번호 숨기기", + "knowledge_enabled": "벡터 매칭", + "knowledge_enabled_hint": "의미적 유사도로 용어와 SQL 예제를 추가 매칭합니다. 비활성화해도 텍스트 매칭은 유지됩니다.", + "table_enabled": "벡터 필터링", + "table_enabled_hint": "질문과 관련된 테이블과 데이터 소스를 선별합니다. 비활성화하면 접근 가능한 모든 후보를 유지합니다.", + "table_selection": "테이블 및 데이터 소스 선택", + "table_count": "테이블 후보 수", + "datasource_count": "데이터 소스 후보 수", + "table_alpha": "테이블 매칭 벡터 가중치", + "keyword_enabled": "키워드 추출", + "table_candidates_hint": "테이블 후보 수는 최종 테이블 수의 상한이 아닙니다. 명시적으로 언급된 테이블과 관계로 보완된 테이블은 유지됩니다. 데이터 소스 후보 수는 모델의 자동 선택 전 벡터 사전 필터링에 적용됩니다.", + "table_weight_hint": "벡터 가중치는 0~1이며 키워드 가중치는 1에서 이 값을 뺀 값입니다.", + "keyword_enabled_hint": "질문에서 키워드를 추출하여 용어와 테이블 매칭에 사용합니다. 벡터 설정과 독립적으로 작동합니다.", + "knowledge_retrieval": "지식 매칭", + "terminology_similarity": "용어 유사도 임계값", + "terminology_top_count": "키워드별 용어 벡터 후보 수", + "data_training_similarity": "SQL 예제 유사도 임계값", + "data_training_top_count": "SQL 예제 벡터 후보 수", + "knowledge_candidates_hint": "후보 수는 벡터 검색에 적용됩니다. 문자 일치와 여러 키워드를 병합하면 최종 지식 수가 늘어날 수 있습니다. 저장한 설정은 새 작업에 적용됩니다.", + "invalid_embedding_parameter": "유사도는 0~1 사이의 유한한 수여야 하며 후보 수는 양의 정수여야 합니다." }, "prompt": { "default_password:": "기본 비밀번호:{msg}", diff --git a/frontend/src/i18n/th-TH.json b/frontend/src/i18n/th-TH.json index 7e623743..9ae16846 100644 --- a/frontend/src/i18n/th-TH.json +++ b/frontend/src/i18n/th-TH.json @@ -69,6 +69,20 @@ "return_to_view": "กลับไปดู" }, "parameter": { + "embedding_scope_hint": "การตั้งค่าเหล่านี้ใช้กับทุกพื้นที่ทำงาน และมีผลกับคำถามและงานเบื้องหลังที่เริ่มใหม่หลังบันทึก", + "embedding_backfill_hint": "เมื่อเปิดจะสร้างและใช้เวกเตอร์ โดยเติมเวกเตอร์ที่ขาดหรือล้าสมัยในเบื้องหลัง เมื่อปิดจะหยุดการสร้างใหม่ แต่งานที่เริ่มแล้วสามารถทำจนเสร็จได้", + "terminology": "คำศัพท์", + "sql_example": "ตัวอย่าง SQL", + "knowledge_type": "ประเภทความรู้", + "similarity_threshold": "เกณฑ์ความคล้ายคลึง", + "vector_candidates": "จำนวนตัวเลือกเวกเตอร์", + "similarity_hint": "ช่วงค่า 0–1 ค่ายิ่งสูงยิ่งจับคู่เข้มงวด", + "terminology_count_hint": "จำนวนตัวเลือกต่อการค้นหาเวกเตอร์ของแต่ละคำสำคัญ เมื่อรวมหลายคำสำคัญและผลการจับคู่ข้อความ จำนวนสุดท้ายอาจเกินค่านี้", + "sql_example_count_hint": "จำนวนตัวเลือกต่อการค้นหาเวกเตอร์ เมื่อรวมผลการจับคู่ข้อความ จำนวนสุดท้ายอาจเกินค่านี้", + "table_count_hint": "จำนวนตัวเลือกที่เก็บจากการกรองเวกเตอร์ ตารางที่ระบุโดยตรงและตารางที่เชื่อมโยงอาจทำให้จำนวนสุดท้ายเพิ่มขึ้น", + "datasource_count_hint": "จำนวนแหล่งข้อมูลที่เก็บจากการกรองเวกเตอร์ก่อนเลือกแหล่งข้อมูลอัตโนมัติ", + "table_alpha_hint": "ช่วงค่า 0–1 ค่ายิ่งสูงยิ่งเน้นการจับคู่เวกเตอร์ น้ำหนักคำสำคัญเท่ากับ 1 ลบค่านี้ ใช้คำนวณคะแนนแบบผสมเมื่อเปิดทั้งการกรองเวกเตอร์ตารางและการดึงคำสำคัญเท่านั้น", + "table_alpha_disabled_hint": "เปิดการดึงคำสำคัญเพื่อปรับน้ำหนักเวกเตอร์ในการจับคู่แบบผสม", "execution_details": "รายละเอียดการทำงาน", "overview": "ภาพรวม", "tokens_required": "Tokens ที่ใช้", @@ -98,7 +112,26 @@ "login_settings": "การตั้งค่าการเข้าสู่ระบบ", "default_login": "วิธีเข้าสู่ระบบเริ่มต้น", "disable_initial_password": "ปิดใช้งานรหัสผ่านเริ่มต้น", - "hide_initial_password": "ซ่อนรหัสผ่านเริ่มต้นบนหน้าเว็บ" + "hide_initial_password": "ซ่อนรหัสผ่านเริ่มต้นบนหน้าเว็บ", + "knowledge_enabled": "การจับคู่เวกเตอร์", + "knowledge_enabled_hint": "จับคู่คำศัพท์และตัวอย่าง SQL เพิ่มเติมด้วยความคล้ายคลึงเชิงความหมาย เมื่อปิดยังคงจับคู่ข้อความได้", + "table_enabled": "การกรองเวกเตอร์", + "table_enabled_hint": "กรองตารางและแหล่งข้อมูลที่เกี่ยวข้องกับคำถาม เมื่อปิดจะเก็บตัวเลือกทั้งหมดที่เข้าถึงได้", + "table_selection": "การเลือกตารางและแหล่งข้อมูล", + "table_count": "จำนวนตารางที่เป็นตัวเลือก", + "datasource_count": "จำนวนแหล่งข้อมูลที่เป็นตัวเลือก", + "table_alpha": "น้ำหนักเวกเตอร์ในการจับคู่ตาราง", + "keyword_enabled": "การดึงคำสำคัญ", + "table_candidates_hint": "จำนวนตารางที่เป็นตัวเลือกไม่ใช่ขีดจำกัดจำนวนตารางสุดท้าย ตารางที่ระบุชัดเจนและตารางที่เพิ่มจากความสัมพันธ์จะยังคงอยู่ จำนวนแหล่งข้อมูลใช้กรองด้วยเวกเตอร์ก่อนให้โมเดลเลือกโดยอัตโนมัติ", + "table_weight_hint": "น้ำหนักเวกเตอร์มีค่าตั้งแต่ 0 ถึง 1 และน้ำหนักคำสำคัญเท่ากับ 1 ลบค่านี้", + "keyword_enabled_hint": "ดึงคำสำคัญจากคำถามเพื่อจับคู่คำศัพท์และตาราง ทำงานแยกจากสวิตช์เวกเตอร์", + "knowledge_retrieval": "การจับคู่ความรู้", + "terminology_similarity": "เกณฑ์ความคล้ายคลึงของคำศัพท์", + "terminology_top_count": "จำนวนคำศัพท์เวกเตอร์ต่อคำสำคัญ", + "data_training_similarity": "เกณฑ์ความคล้ายคลึงของตัวอย่าง SQL", + "data_training_top_count": "จำนวนตัวอย่าง SQL เวกเตอร์", + "knowledge_candidates_hint": "จำนวนนี้ใช้กับการค้นคืนเวกเตอร์ การรวมผลตรงตัวและหลายคำสำคัญอาจเพิ่มจำนวนความรู้ที่ส่งให้โมเดล การตั้งค่าที่บันทึกมีผลกับงานใหม่", + "invalid_embedding_parameter": "ความคล้ายคลึงต้องเป็นจำนวนจำกัดระหว่าง 0 ถึง 1 และจำนวนรายการต้องเป็นจำนวนเต็มบวก" }, "prompt": { "default_password": "รหัสผ่านเริ่มต้น: {msg}", diff --git a/frontend/src/i18n/zh-CN.json b/frontend/src/i18n/zh-CN.json index 81ecd3a1..94c6d876 100644 --- a/frontend/src/i18n/zh-CN.json +++ b/frontend/src/i18n/zh-CN.json @@ -69,6 +69,20 @@ "return_to_view": "返回查看" }, "parameter": { + "embedding_scope_hint": "以下设置适用于所有工作空间，保存后对新发起的问数和后台任务生效。", + "embedding_backfill_hint": "开启后生成并使用向量，缺失或失效向量会在后台自动补齐；关闭后停止新向量生成，已开始的任务可继续完成。", + "terminology": "术语", + "sql_example": "SQL 示例", + "knowledge_type": "知识类型", + "similarity_threshold": "相似度阈值", + "vector_candidates": "向量候选数量", + "similarity_hint": "取值范围 0～1，越高匹配越严格。", + "terminology_count_hint": "每次关键词向量检索的候选数量；合并多个关键词和文字匹配后，最终数量可能超过此值。", + "sql_example_count_hint": "向量检索的候选数量；合并文字匹配后，最终数量可能超过此值。", + "table_count_hint": "向量筛选的候选数量；明确提及的表和关系补全表可能使最终数量超过此值。", + "datasource_count_hint": "自动选择数据源前，向量筛选保留的数据源候选数量。", + "table_alpha_hint": "取值范围 0～1，越大越偏重向量匹配，关键词权重为 1 减去该值。仅在表向量筛选和关键词提取均开启时参与混合评分。", + "table_alpha_disabled_hint": "开启关键词提取后，可调整混合匹配的向量权重。", "execution_details": "执行详情", "overview": "概览", "tokens_required": "消耗 Tokens", @@ -98,7 +112,26 @@ "login_settings": "登录设置", "default_login": "默认登录方式", "disable_initial_password": "禁用初始密码", - "hide_initial_password": "页面隐藏初始密码" + "hide_initial_password": "页面隐藏初始密码", + "knowledge_enabled": "向量匹配", + "knowledge_enabled_hint": "通过语义相似度补充匹配术语和 SQL 示例，关闭后仍保留文字匹配。", + "table_enabled": "向量筛选", + "table_enabled_hint": "根据问题筛选相关表和数据源，关闭后保留全部可访问候选。", + "table_selection": "表与数据源筛选", + "table_count": "表候选数量", + "datasource_count": "数据源候选数量", + "table_alpha": "表匹配向量权重", + "keyword_enabled": "关键词提取", + "table_candidates_hint": "表候选数量不是最终表数上限：明确提及的表和关系补全表会保留。数据源候选数量用于模型自动选数据源前的向量预筛选。", + "table_weight_hint": "向量权重范围为 0～1，关键词权重为 1 减去该值。", + "keyword_enabled_hint": "提取问题中的关键词，用于术语匹配和表匹配。独立于向量开关生效。", + "knowledge_retrieval": "知识匹配", + "terminology_similarity": "术语相似度阈值", + "terminology_top_count": "每个关键词的术语向量候选数量", + "data_training_similarity": "SQL 示例相似度阈值", + "data_training_top_count": "SQL 示例向量候选数量", + "knowledge_candidates_hint": "候选数量仅作用于向量召回。合并字面匹配和多个关键词后，最终注入的知识数量可能增加。保存后对新任务生效。", + "invalid_embedding_parameter": "相似度必须是 0～1 的有限数值，候选数量必须是正整数。" }, "prompt": { "default_password": "默认密码：{msg}", diff --git a/frontend/src/i18n/zh-TW.json b/frontend/src/i18n/zh-TW.json index fc2abbf2..252f36bc 100644 --- a/frontend/src/i18n/zh-TW.json +++ b/frontend/src/i18n/zh-TW.json @@ -69,6 +69,20 @@ "return_to_view": "返回查看" }, "parameter": { + "embedding_scope_hint": "以下設定適用於所有工作空間，儲存後對新發起的問數和背景任務生效。", + "embedding_backfill_hint": "開啟後產生並使用向量，缺失或失效向量會在背景自動補齊；關閉後停止新的向量產生，已開始的任務可繼續完成。", + "terminology": "術語", + "sql_example": "SQL 範例", + "knowledge_type": "知識類型", + "similarity_threshold": "相似度閾值", + "vector_candidates": "向量候選數量", + "similarity_hint": "取值範圍 0～1，越高匹配越嚴格。", + "terminology_count_hint": "每次關鍵詞向量檢索的候選數量；合併多個關鍵詞和文字匹配後，最終數量可能超過此值。", + "sql_example_count_hint": "向量檢索的候選數量；合併文字匹配後，最終數量可能超過此值。", + "table_count_hint": "向量篩選的候選數量；明確提及的表和關係補全表可能使最終數量超過此值。", + "datasource_count_hint": "自動選擇資料來源前，向量篩選保留的資料來源候選數量。", + "table_alpha_hint": "取值範圍 0～1，越大越偏重向量匹配，關鍵詞權重為 1 減去該值。僅在表向量篩選和關鍵詞擷取均開啟時參與混合評分。", + "table_alpha_disabled_hint": "開啟關鍵詞擷取後，可調整混合匹配的向量權重。", "execution_details": "執行詳情", "overview": "概覽", "tokens_required": "消耗 Tokens", @@ -98,7 +112,26 @@ "login_settings": "登入設定", "default_login": "預設登入方式", "disable_initial_password": "禁用初始密碼", - "hide_initial_password": "頁面隱藏初始密碼" + "hide_initial_password": "頁面隱藏初始密碼", + "knowledge_enabled": "向量匹配", + "knowledge_enabled_hint": "透過語意相似度補充匹配術語和 SQL 範例，關閉後仍保留文字匹配。", + "table_enabled": "向量篩選", + "table_enabled_hint": "根據問題篩選相關表和資料來源，關閉後保留全部可存取的候選。", + "table_selection": "表與資料來源篩選", + "table_count": "表候選數量", + "datasource_count": "資料來源候選數量", + "table_alpha": "表比對向量權重", + "keyword_enabled": "關鍵詞擷取", + "table_candidates_hint": "表候選數量不是最終表數上限：明確提及的表和關聯補全表會保留。資料來源候選數量用於模型自動選擇資料來源前的向量預篩選。", + "table_weight_hint": "向量權重範圍為 0～1，關鍵詞權重為 1 減去此值。", + "keyword_enabled_hint": "擷取問題中的關鍵詞，用於術語匹配和表匹配。獨立於向量開關生效。", + "knowledge_retrieval": "知識匹配", + "terminology_similarity": "術語相似度閾值", + "terminology_top_count": "每個關鍵詞的術語向量候選數量", + "data_training_similarity": "SQL 範例相似度閾值", + "data_training_top_count": "SQL 範例向量候選數量", + "knowledge_candidates_hint": "候選數量僅作用於向量召回。合併字面匹配和多個關鍵詞後，最終注入的知識數量可能增加。儲存後對新任務生效。", + "invalid_embedding_parameter": "相似度必須是 0～1 的有限數值，候選數量必須是正整數。" }, "prompt": { "default_password": "預設密碼：{msg}", diff --git a/frontend/src/views/system/parameter/index.vue b/frontend/src/views/system/parameter/index.vue index 242c279e..643e65c7 100644 --- a/frontend/src/views/system/parameter/index.vue +++ b/frontend/src/views/system/parameter/index.vue @@ -6,6 +6,43 @@ import PlatformParam from './xpack/PlatformParam.vue' import { request } from '@/utils/request' import { formatArg } from '@/utils/utils' const { t } = useI18n() +const knowledgeFields = [ + { key: 'embedding.terminology_similarity', label: 'terminology_similarity', integer: false }, + { key: 'embedding.terminology_top_count', label: 'terminology_top_count', integer: true }, + { key: 'embedding.data_training_similarity', label: 'data_training_similarity', integer: false }, + { key: 'embedding.data_training_top_count', label: 'data_training_top_count', integer: true }, +] as const + +const knowledgeRows = [ + { label: 'terminology', similarity: knowledgeFields[0], count: knowledgeFields[1] }, + { label: 'sql_example', similarity: knowledgeFields[2], count: knowledgeFields[3] }, +] as const + +const tableFields = [ + { key: 'embedding.table_count', label: 'table_count', integer: true }, + { key: 'embedding.datasource_count', label: 'datasource_count', integer: true }, + { key: 'embedding.table_alpha', label: 'table_alpha', integer: false }, +] as const +const booleanFields = [ + 'embedding.enabled', + 'embedding.keyword_enabled', + 'embedding.table_enabled', +] as const +type NumericParameterKey = + (typeof knowledgeFields)[number]['key'] | (typeof tableFields)[number]['key'] +type BooleanParameterKey = (typeof booleanFields)[number] +type EmbeddingParameterForm = Partial< + Record & Record +> +interface ParameterValue { + pkey: string + pval: string | number | boolean | null +} +const embeddingForm = reactive({}) +const isBooleanParameter = (key: string): key is BooleanParameterKey => + booleanFields.some((field) => field === key) +const isNumericParameter = (key: string): key is NumericParameterKey => + [...knowledgeFields, ...tableFields].some((field) => field.key === key) const state = reactive({ parameterForm: reactive({ @@ -19,24 +56,27 @@ const state = reactive({ }) provide('parameterForm', state.parameterForm) const loadData = () => { - request.get('/system/parameter').then((res: any) => { + request.get('/system/parameter').then((res: ParameterValue[]) => { if (res) { - res.forEach((item: any) => { - if ( + res.forEach((item) => { + if (isBooleanParameter(item.pkey)) { + embeddingForm[item.pkey] = item.pval === true || item.pval === 'true' + } else if (isNumericParameter(item.pkey)) { + embeddingForm[item.pkey] = Number(item.pval) + } else if ( item.pkey?.startsWith('chat') || item.pkey?.startsWith('login') || item.pkey?.startsWith('platform') ) { if (item.pkey === 'chat.sqlbot_name') { - if (item.pval && item.pval.trim().length > 0) { + if (typeof item.pval === 'string' && item.pval.trim().length > 0) { state.parameterForm[item.pkey] = item.pval } } else { - state.parameterForm[item.pkey] = formatArg(item.pval) + state.parameterForm[item.pkey] = formatArg(String(item.pval)) } } }) - console.log(state.parameterForm) } }) } @@ -66,19 +106,29 @@ const beforeChange = (): Promise => { }) } const buildParam = () => { - const changedItemArray = Object.keys(state.parameterForm).map((key: string) => { - return { - pkey: key, - pval: Object.prototype.hasOwnProperty.call(state.parameterForm, 'key') - ? state.parameterForm[key].toString() - : state.parameterForm[key], - } - }) + const changedItemArray = Object.entries({ ...state.parameterForm, ...embeddingForm }).map( + ([pkey, pval]) => ({ + pkey, + pval: String(pval), + }) + ) const formData = new FormData() formData.append('data', JSON.stringify(unref(changedItemArray))) return formData } const saveHandler = () => { + const invalid = [...knowledgeFields, ...tableFields].some(({ key, integer }) => { + const value = embeddingForm[key] + return ( + typeof value !== 'number' || + !Number.isFinite(value) || + (integer ? !Number.isInteger(value) || value <= 0 : value < 0 || value > 1) + ) + }) + if (invalid || booleanFields.some((key) => typeof embeddingForm[key] !== 'boolean')) { + ElMessage.error(t('parameter.invalid_embedding_parameter')) + return + } const param = buildParam() request .post('/system/parameter', param, { @@ -88,6 +138,7 @@ const saveHandler = () => { }) .then(() => { ElMessage.success(t('common.save_success')) + loadData() }) } onMounted(() => { @@ -199,9 +250,134 @@ onMounted(() => { +

{{ t('parameter.embedding_scope_hint') }}

+
+
+
{{ t('parameter.knowledge_retrieval') }}
+
+ {{ t('parameter.knowledge_enabled') }} + + + +
+
+

{{ t('parameter.knowledge_enabled_hint') }}

+
+ + + + + + + + + + + + + + + +
{{ t('parameter.knowledge_type') }} + + {{ t('parameter.similarity_threshold') }} + + + + + + + {{ t('parameter.vector_candidates') }}
{{ t(`parameter.${row.label}`) }} + + +
+ + + + + + +
+
+
+
+
+
+
{{ t('parameter.table_selection') }}
+
+ {{ t('parameter.table_enabled') }} + + + +
+
+

{{ t('parameter.table_enabled_hint') }}

+
+
+
+ {{ t(`parameter.${field.label}`) }} + + + + + +
+
+ +
+

+ {{ t('parameter.table_alpha_disabled_hint') }} +

+
+
+
+
+
+
{{ t('parameter.keyword_enabled') }}
+
+ +
+
+

{{ t('parameter.keyword_enabled_hint') }}

+
-
+
{{ t('common.save') }}
@@ -230,6 +406,83 @@ onMounted(() => { line-height: 28px; margin-bottom: 16px; } + .embedding-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 16px 24px; + } + .embedding-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + } + .embedding-toggle, + .embedding-label { + display: inline-flex; + align-items: center; + gap: 8px; + } + .embedding-toggle { + flex-shrink: 0; + font-size: 14px; + } + .embedding-label .ed-icon { + flex-shrink: 0; + color: var(--ed-text-color-secondary); + } + .knowledge-table-wrapper { + overflow-x: auto; + margin-top: 16px; + } + .knowledge-table { + width: 100%; + border-collapse: collapse; + text-align: left; + font-size: 14px; + + th, + td { + padding: 12px 16px; + font-weight: 400; + } + thead { + background: var(--ed-fill-color-light); + color: var(--ed-text-color-secondary); + } + tbody tr + tr { + border-top: 1px solid var(--ed-border-color-lighter); + } + th:first-child { + width: 24%; + min-width: 110px; + } + } + .save { + position: sticky; + bottom: 0; + z-index: 2; + margin-top: 16px; + padding: 16px 0; + background: var(--ed-bg-color); + border-top: 1px solid var(--ed-border-color-lighter); + } + .hint.embedding-scope { + margin-top: 24px; + } + .card-container .card.embedding-card { + .embedding-header .card-title { + width: auto; + } + .card-item { + width: 100%; + } + } + .hint { + color: var(--ed-text-color-secondary); + margin: 8px 0 0; + font-size: 12px; + } .card-container { .card { width: 100%; @@ -274,5 +527,13 @@ onMounted(() => { } } } + @media (max-width: 960px) { + .embedding-grid { + grid-template-columns: 1fr; + } + .embedding-header { + flex-wrap: wrap; + } + } }