[AC-MIGRATION] feat(db): 新增用户记忆迁移和工具脚本
- 新增 003_user_memories 迁移脚本支持用户记忆表 - 新增 clear_kb_vectors 脚本用于清理知识库向量 - 新增 svg 资源目录
This commit is contained in:
MerCry
parent
a6276522c8
commit
60e16d65c9
|
|
@ -0,0 +1,75 @@
|
|||
"""
|
||||
Database Migration: User Memories Table.
|
||||
[AC-IDMP-14] 用户级记忆滚动摘要表
|
||||
|
||||
创建时间: 2025-03-08
|
||||
变更说明:
|
||||
- 新增 user_memories 表用于存储滚动摘要与事实/偏好/未解决问题
|
||||
|
||||
执行方式:
|
||||
- SQLModel 会自动创建表（通过 init_db）
|
||||
- 此脚本用于手动迁移或回滚
|
||||
|
||||
SQL DDL:
|
||||
```sql
|
||||
CREATE TABLE user_memories (
|
||||
id UUID PRIMARY KEY,
|
||||
tenant_id VARCHAR NOT NULL,
|
||||
user_id VARCHAR NOT NULL,
|
||||
summary TEXT,
|
||||
facts JSON,
|
||||
preferences JSON,
|
||||
open_issues JSON,
|
||||
summary_version INTEGER NOT NULL DEFAULT 1,
|
||||
last_turn_id VARCHAR,
|
||||
expires_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX ix_user_memories_tenant_user ON user_memories(tenant_id, user_id);
|
||||
CREATE INDEX ix_user_memories_tenant_user_updated ON user_memories(tenant_id, user_id, updated_at);
|
||||
```
|
||||
|
||||
回滚 SQL:
|
||||
```sql
|
||||
DROP TABLE IF EXISTS user_memories;
|
||||
```
|
||||
"""
|
||||
|
||||
USER_MEMORIES_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS user_memories (
|
||||
id UUID PRIMARY KEY,
|
||||
tenant_id VARCHAR NOT NULL,
|
||||
user_id VARCHAR NOT NULL,
|
||||
summary TEXT,
|
||||
facts JSON,
|
||||
preferences JSON,
|
||||
open_issues JSON,
|
||||
summary_version INTEGER NOT NULL DEFAULT 1,
|
||||
last_turn_id VARCHAR,
|
||||
expires_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT NOW()
|
||||
);
|
||||
"""
|
||||
|
||||
USER_MEMORIES_INDEXES = """
|
||||
CREATE INDEX IF NOT EXISTS ix_user_memories_tenant_user ON user_memories(tenant_id, user_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_user_memories_tenant_user_updated ON user_memories(tenant_id, user_id, updated_at);
|
||||
"""
|
||||
|
||||
USER_MEMORIES_ROLLBACK = """
|
||||
DROP TABLE IF EXISTS user_memories;
|
||||
"""
|
||||
|
||||
|
||||
async def upgrade(conn):
|
||||
"""执行迁移"""
|
||||
await conn.execute(USER_MEMORIES_DDL)
|
||||
await conn.execute(USER_MEMORIES_INDEXES)
|
||||
|
||||
|
||||
async def downgrade(conn):
|
||||
"""回滚迁移"""
|
||||
await conn.execute(USER_MEMORIES_ROLLBACK)
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
"""
|
||||
Script to cleanup vector data for a specific knowledge base.
|
||||
Clears the Qdrant collection for the given KB ID, allowing re-indexing.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "Q:\\agentProject\\ai-robot-core\\ai-service")
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.qdrant_client import get_qdrant_client
|
||||
from app.core.database import get_session
|
||||
from app.models.entities import KnowledgeBase, Document
|
||||
from sqlalchemy import select
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def get_knowledge_base_info(kb_id: str) -> dict | None:
|
||||
"""Get knowledge base information from database."""
|
||||
async for session in get_session():
|
||||
stmt = select(KnowledgeBase).where(KnowledgeBase.id == kb_id)
|
||||
result = await session.execute(stmt)
|
||||
kb = result.scalar_one_or_none()
|
||||
|
||||
if kb:
|
||||
doc_stmt = select(Document).where(Document.kb_id == kb_id)
|
||||
doc_result = await session.execute(doc_stmt)
|
||||
documents = doc_result.scalars().all()
|
||||
|
||||
return {
|
||||
"id": str(kb.id),
|
||||
"tenant_id": kb.tenant_id,
|
||||
"name": kb.name,
|
||||
"doc_count": len(documents),
|
||||
"document_ids": [str(doc.id) for doc in documents]
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
async def list_kb_collections(tenant_id: str, kb_id: str) -> list[str]:
|
||||
"""List all collections that might be related to the KB."""
|
||||
client = await get_qdrant_client()
|
||||
qdrant = await client.get_client()
|
||||
|
||||
collections = await qdrant.get_collections()
|
||||
all_names = [c.name for c in collections.collections]
|
||||
|
||||
safe_tenant = tenant_id.replace('@', '_')
|
||||
safe_kb = kb_id.replace('-', '_')[:8]
|
||||
|
||||
matching = [
|
||||
name for name in all_names
|
||||
if safe_kb in name or kb_id.replace('-', '')[:8] in name.replace('_', '')
|
||||
]
|
||||
|
||||
return matching
|
||||
|
||||
|
||||
async def clear_kb_vector_data(tenant_id: str, kb_id: str, delete_docs: bool = False) -> bool:
|
||||
"""
|
||||
Clear vector data for a specific knowledge base.
|
||||
|
||||
Args:
|
||||
tenant_id: Tenant identifier
|
||||
kb_id: Knowledge base ID
|
||||
delete_docs: Whether to also delete document records from database
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
client = await get_qdrant_client()
|
||||
qdrant = await client.get_client()
|
||||
|
||||
collection_name = client.get_kb_collection_name(tenant_id, kb_id)
|
||||
|
||||
try:
|
||||
exists = await qdrant.collection_exists(collection_name)
|
||||
if exists:
|
||||
await qdrant.delete_collection(collection_name=collection_name)
|
||||
logger.info(f"Deleted Qdrant collection: {collection_name}")
|
||||
else:
|
||||
logger.info(f"Collection {collection_name} does not exist")
|
||||
|
||||
if delete_docs:
|
||||
async for session in get_session():
|
||||
doc_stmt = select(Document).where(Document.kb_id == kb_id)
|
||||
doc_result = await session.execute(doc_stmt)
|
||||
documents = doc_result.scalars().all()
|
||||
|
||||
for doc in documents:
|
||||
await session.delete(doc)
|
||||
|
||||
stmt = select(KnowledgeBase).where(KnowledgeBase.id == kb_id)
|
||||
result = await session.execute(stmt)
|
||||
kb = result.scalar_one_or_none()
|
||||
if kb:
|
||||
kb.doc_count = 0
|
||||
kb.updated_at = datetime.utcnow()
|
||||
|
||||
await session.commit()
|
||||
logger.info(f"Deleted {len(documents)} document records from database")
|
||||
break
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear KB vector data: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def main(kb_id: str, delete_docs: bool = False):
|
||||
"""Main function to clear KB vector data."""
|
||||
logger.info(f"Starting cleanup for knowledge base: {kb_id}")
|
||||
|
||||
kb_info = await get_knowledge_base_info(kb_id)
|
||||
|
||||
if not kb_info:
|
||||
logger.error(f"Knowledge base not found: {kb_id}")
|
||||
return False
|
||||
|
||||
logger.info(f"Found knowledge base:")
|
||||
logger.info(f" - ID: {kb_info['id']}")
|
||||
logger.info(f" - Name: {kb_info['name']}")
|
||||
logger.info(f" - Tenant: {kb_info['tenant_id']}")
|
||||
logger.info(f" - Document count: {kb_info['doc_count']}")
|
||||
|
||||
matching_collections = await list_kb_collections(kb_info['tenant_id'], kb_id)
|
||||
if matching_collections:
|
||||
logger.info(f" - Related collections: {matching_collections}")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("WARNING: This will delete all vector data for this knowledge base!")
|
||||
print(f"Collection to delete: kb_{kb_info['tenant_id'].replace('@', '_')}_{kb_id.replace('-', '_')[:8]}")
|
||||
if delete_docs:
|
||||
print("Document records in database will also be deleted!")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
confirm = input("Continue? (yes/no): ")
|
||||
if confirm.lower() != "yes":
|
||||
print("Cancelled")
|
||||
return False
|
||||
|
||||
success = await clear_kb_vector_data(
|
||||
tenant_id=kb_info['tenant_id'],
|
||||
kb_id=kb_id,
|
||||
delete_docs=delete_docs
|
||||
)
|
||||
|
||||
if success:
|
||||
logger.info(f"Successfully cleared vector data for KB: {kb_id}")
|
||||
logger.info("You can now re-index the knowledge base documents.")
|
||||
else:
|
||||
logger.error(f"Failed to clear vector data for KB: {kb_id}")
|
||||
|
||||
return success
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
parser = argparse.ArgumentParser(description="Clear vector data for a knowledge base")
|
||||
parser.add_argument("kb_id", help="Knowledge base ID to clear")
|
||||
parser.add_argument("--delete-docs", action="store_true",
|
||||
help="Also delete document records from database")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
asyncio.run(main(args.kb_id, args.delete_docs))
|
||||
|
|
@ -0,0 +1 @@
|
|||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg class="icon" width="200px" height="200.00px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"><path fill="#1296db" d="M894.1 355.6h-1.7C853 177.6 687.6 51.4 498.1 54.9S148.2 190.5 115.9 369.7c-35.2 5.6-61.1 36-61.1 71.7v143.4c0.9 40.4 34.3 72.5 74.7 71.7 21.7-0.3 42.2-10 56-26.7 33.6 84.5 99.9 152 183.8 187 1.1-2 2.3-3.9 3.7-5.7 0.9-1.5 2.4-2.6 4.1-3 1.3 0 2.5 0.5 3.6 1.2a318.46 318.46 0 0 1-105.3-187.1c-5.1-44.4 24.1-85.4 67.6-95.2 64.3-11.7 128.1-24.7 192.4-35.9 37.9-5.3 70.4-29.8 85.7-64.9 6.8-15.9 11-32.8 12.5-50 0.5-3.1 2.9-5.6 5.9-6.2 3.1-0.7 6.4 0.5 8.2 3l1.7-1.1c25.4 35.9 74.7 114.4 82.7 197.2 8.2 94.8 3.7 160-71.4 226.5-1.1 1.1-1.7 2.6-1.7 4.1 0.1 2 1.1 3.8 2.8 4.8h4.8l3.2-1.8c75.6-40.4 132.8-108.2 159.9-189.5 11.4 16.1 28.5 27.1 47.8 30.8C846 783.9 716.9 871.6 557.2 884.9c-12-28.6-42.5-44.8-72.9-38.6-33.6 5.4-56.6 37-51.2 70.6 4.4 27.6 26.8 48.8 54.5 51.6 30.6 4.6 60.3-13 70.8-42.2 184.9-14.5 333.2-120.8 364.2-286.9 27.8-10.8 46.3-37.4 46.6-67.2V428.7c-0.1-19.5-8.1-38.2-22.3-51.6-14.5-13.8-33.8-21.4-53.8-21.3l1-0.2zM825.9 397c-71.1-176.9-272.1-262.7-449-191.7-86.8 34.9-155.7 103.4-191 190-2.5-2.8-5.2-5.4-8-7.9 25.3-154.6 163.8-268.6 326.8-269.2s302.3 112.6 328.7 267c-2.9 3.8-5.4 7.7-7.5 11.8z" /></svg>
|
||||
|
After Width: | Height: | Size: 1.4 KiB |
|
|
@ -0,0 +1 @@
|
|||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg class="icon" width="200px" height="200.00px" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"><path fill="#1296db" d="M573.9 516.2L512 640l-61.9-123.8C232 546.4 64 733.6 64 960h896c0-226.4-168-413.6-386.1-443.8zM480 384h64c17.7 0 32.1 14.4 32.1 32.1 0 17.7-14.4 32.1-32.1 32.1h-64c-11.9 0-22.3-6.5-27.8-16.1H356c34.9 48.5 91.7 80 156 80 106 0 192-86 192-192s-86-192-192-192-192 86-192 192c0 28.5 6.2 55.6 17.4 80h114.8c5.5-9.6 15.9-16.1 27.8-16.1z" /><path fill="#1296db" d="M272 432.1h84c-4.2-5.9-8.1-12-11.7-18.4-2.3-4.1-4.4-8.3-6.4-12.5-0.2-0.4-0.4-0.7-0.5-1.1H288c-8.8 0-16-7.2-16-16v-48.4c0-64.1 25-124.3 70.3-169.6S447.9 95.8 512 95.8s124.3 25 169.7 70.3c38.3 38.3 62.1 87.2 68.5 140.2-8.4 4-14.2 12.5-14.2 22.4v78.6c0 13.7 11.1 24.8 24.8 24.8h14.6c13.7 0 24.8-11.1 24.8-24.8v-78.6c0-11.3-7.6-20.9-18-23.8-6.9-60.9-33.9-117.4-78-161.3C652.9 92.1 584.6 63.9 512 63.9s-140.9 28.3-192.3 79.6C268.3 194.8 240 263.1 240 335.7v64.4c0 17.7 14.3 32 32 32z" /></svg>
|
||||
|
After Width: | Height: | Size: 1.1 KiB |
|
|
@ -0,0 +1 @@
|
|||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1773157868702" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="12129" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><path d="M657.066667 558.933333c-34.133333 25.6-76.8 38.4-123.733334 38.4s-85.333333-12.8-123.733333-38.4c0 4.266667-4.266667 4.266667-8.533333 4.266667C315.733333 614.4 256 708.266667 256 810.666667c0 12.8-8.533333 21.333333-21.333333 21.333333S213.333333 823.466667 213.333333 810.666667c0-119.466667 64-226.133333 166.4-281.6-38.4-38.4-59.733333-89.6-59.733333-145.066667 0-119.466667 93.866667-213.333333 213.333333-213.333333s213.333333 93.866667 213.333334 213.333333c0 55.466667-21.333333 106.666667-59.733334 145.066667 102.4 55.466667 166.4 162.133333 166.4 281.6 0 12.8-8.533333 21.333333-21.333333 21.333333s-21.333333-8.533333-21.333333-21.333333c0-102.4-59.733333-196.266667-149.333334-247.466667 0 0-4.266667 0-4.266666-4.266667z m-123.733334-4.266666c93.866667 0 170.666667-76.8 170.666667-170.666667s-76.8-170.666667-170.666667-170.666667-170.666667 76.8-170.666666 170.666667 76.8 170.666667 170.666666 170.666667z" p-id="12130" fill="#1296db"></path></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
Loading…
Reference in New Issue