283 lines
8.2 KiB
Python
283 lines
8.2 KiB
Python
"""
|
||
Segment Humanizer for Mid Platform.
|
||
[AC-MARH-10] 分段策略组件（语义/长度切分）。
|
||
[AC-MARH-11] delay 策略租户化配置。
|
||
|
||
将文本按语义/长度切分为 segments，并生成拟人化 delay。
|
||
"""
|
||
|
||
import logging
|
||
import re
|
||
import uuid
|
||
from dataclasses import dataclass, field
|
||
from typing import Any
|
||
|
||
from app.models.mid.schemas import Segment, SegmentStats
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
DEFAULT_MIN_DELAY_MS = 50
|
||
DEFAULT_MAX_DELAY_MS = 500
|
||
DEFAULT_SEGMENT_MIN_LENGTH = 10
|
||
DEFAULT_SEGMENT_MAX_LENGTH = 200
|
||
|
||
|
||
@dataclass
|
||
class HumanizeConfig:
|
||
"""拟人化配置。"""
|
||
enabled: bool = True
|
||
min_delay_ms: int = DEFAULT_MIN_DELAY_MS
|
||
max_delay_ms: int = DEFAULT_MAX_DELAY_MS
|
||
length_bucket_strategy: str = "simple"
|
||
segment_min_length: int = DEFAULT_SEGMENT_MIN_LENGTH
|
||
segment_max_length: int = DEFAULT_SEGMENT_MAX_LENGTH
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
return {
|
||
"enabled": self.enabled,
|
||
"min_delay_ms": self.min_delay_ms,
|
||
"max_delay_ms": self.max_delay_ms,
|
||
"length_bucket_strategy": self.length_bucket_strategy,
|
||
"segment_min_length": self.segment_min_length,
|
||
"segment_max_length": self.segment_max_length,
|
||
}
|
||
|
||
@classmethod
|
||
def from_dict(cls, data: dict[str, Any]) -> "HumanizeConfig":
|
||
return cls(
|
||
enabled=data.get("enabled", True),
|
||
min_delay_ms=data.get("min_delay_ms", DEFAULT_MIN_DELAY_MS),
|
||
max_delay_ms=data.get("max_delay_ms", DEFAULT_MAX_DELAY_MS),
|
||
length_bucket_strategy=data.get("length_bucket_strategy", "simple"),
|
||
segment_min_length=data.get("segment_min_length", DEFAULT_SEGMENT_MIN_LENGTH),
|
||
segment_max_length=data.get("segment_max_length", DEFAULT_SEGMENT_MAX_LENGTH),
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class LengthBucket:
|
||
"""长度区间与对应 delay。"""
|
||
min_length: int
|
||
max_length: int
|
||
delay_ms: int
|
||
|
||
|
||
DEFAULT_LENGTH_BUCKETS = [
|
||
LengthBucket(min_length=0, max_length=20, delay_ms=100),
|
||
LengthBucket(min_length=20, max_length=50, delay_ms=200),
|
||
LengthBucket(min_length=50, max_length=100, delay_ms=300),
|
||
LengthBucket(min_length=100, max_length=200, delay_ms=400),
|
||
LengthBucket(min_length=200, max_length=10000, delay_ms=500),
|
||
]
|
||
|
||
|
||
class SegmentHumanizer:
|
||
"""
|
||
[AC-MARH-10, AC-MARH-11] 分段拟人化组件。
|
||
|
||
Features:
|
||
- 按语义/长度切分文本
|
||
- 生成拟人化 delay
|
||
- 支持租户配置覆盖
|
||
- 输出 segment_stats 统计
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
config: HumanizeConfig | None = None,
|
||
length_buckets: list[LengthBucket] | None = None,
|
||
):
|
||
self._config = config or HumanizeConfig()
|
||
self._length_buckets = length_buckets or DEFAULT_LENGTH_BUCKETS
|
||
|
||
def humanize(
|
||
self,
|
||
text: str,
|
||
override_config: HumanizeConfig | None = None,
|
||
) -> tuple[list[Segment], SegmentStats]:
|
||
"""
|
||
[AC-MARH-10] 将文本转换为拟人化分段。
|
||
|
||
Args:
|
||
text: 输入文本
|
||
override_config: 租户覆盖配置
|
||
|
||
Returns:
|
||
Tuple of (segments, segment_stats)
|
||
"""
|
||
config = override_config or self._config
|
||
|
||
if not config.enabled:
|
||
segments = [Segment(
|
||
segment_id=str(uuid.uuid4()),
|
||
text=text,
|
||
delay_after=0,
|
||
)]
|
||
stats = SegmentStats(
|
||
segment_count=1,
|
||
avg_segment_length=len(text),
|
||
humanize_strategy="disabled",
|
||
)
|
||
return segments, stats
|
||
|
||
raw_segments = self._split_text(text, config)
|
||
segments = []
|
||
|
||
for i, seg_text in enumerate(raw_segments):
|
||
is_last = i == len(raw_segments) - 1
|
||
delay_after = 0 if is_last else self._calculate_delay(seg_text, config)
|
||
|
||
segments.append(Segment(
|
||
segment_id=str(uuid.uuid4()),
|
||
text=seg_text,
|
||
delay_after=delay_after,
|
||
))
|
||
|
||
total_length = sum(len(s.text) for s in segments)
|
||
avg_length = total_length / len(segments) if segments else 0.0
|
||
|
||
stats = SegmentStats(
|
||
segment_count=len(segments),
|
||
avg_segment_length=avg_length,
|
||
humanize_strategy=config.length_bucket_strategy,
|
||
)
|
||
|
||
logger.info(
|
||
f"[AC-MARH-10] Humanized text: segments={len(segments)}, "
|
||
f"avg_length={avg_length:.1f}, strategy={config.length_bucket_strategy}"
|
||
)
|
||
|
||
return segments, stats
|
||
|
||
def _split_text(
|
||
self,
|
||
text: str,
|
||
config: HumanizeConfig,
|
||
) -> list[str]:
|
||
"""切分文本。"""
|
||
if config.length_bucket_strategy == "semantic":
|
||
return self._split_semantic(text, config)
|
||
else:
|
||
return self._split_simple(text, config)
|
||
|
||
def _split_simple(
|
||
self,
|
||
text: str,
|
||
config: HumanizeConfig,
|
||
) -> list[str]:
|
||
"""简单切分：按段落。"""
|
||
paragraphs = re.split(r'\n\s*\n', text.strip())
|
||
segments = []
|
||
|
||
for para in paragraphs:
|
||
para = para.strip()
|
||
if not para:
|
||
continue
|
||
|
||
if len(para) <= config.segment_max_length:
|
||
segments.append(para)
|
||
else:
|
||
sub_segments = self._split_by_length(para, config.segment_max_length)
|
||
segments.extend(sub_segments)
|
||
|
||
if not segments:
|
||
segments = [text.strip()]
|
||
|
||
return [s for s in segments if s.strip()]
|
||
|
||
def _split_semantic(
|
||
self,
|
||
text: str,
|
||
config: HumanizeConfig,
|
||
) -> list[str]:
|
||
"""语义切分：按句子边界。"""
|
||
sentence_endings = re.compile(r'([。！？.!?]+)')
|
||
parts = sentence_endings.split(text.strip())
|
||
|
||
sentences = []
|
||
current = ""
|
||
for i, part in enumerate(parts):
|
||
current += part
|
||
if sentence_endings.match(part):
|
||
sentences.append(current.strip())
|
||
current = ""
|
||
|
||
if current.strip():
|
||
sentences.append(current.strip())
|
||
|
||
if not sentences:
|
||
sentences = [text.strip()]
|
||
|
||
segments = []
|
||
current_segment = ""
|
||
|
||
for sentence in sentences:
|
||
if len(current_segment) + len(sentence) <= config.segment_max_length:
|
||
current_segment += sentence
|
||
else:
|
||
if current_segment:
|
||
segments.append(current_segment)
|
||
current_segment = sentence
|
||
|
||
if current_segment:
|
||
segments.append(current_segment)
|
||
|
||
return [s for s in segments if s.strip()]
|
||
|
||
def _split_by_length(
|
||
self,
|
||
text: str,
|
||
max_length: int,
|
||
) -> list[str]:
|
||
"""按长度切分。"""
|
||
segments = []
|
||
remaining = text
|
||
|
||
while remaining:
|
||
if len(remaining) <= max_length:
|
||
segments.append(remaining.strip())
|
||
break
|
||
|
||
split_pos = max_length
|
||
for i in range(max_length - 1, max(0, max_length - 20), -1):
|
||
if remaining[i] in '，,；;：: ':
|
||
split_pos = i + 1
|
||
break
|
||
|
||
segments.append(remaining[:split_pos].strip())
|
||
remaining = remaining[split_pos:]
|
||
|
||
return [s for s in segments if s.strip()]
|
||
|
||
def _calculate_delay(
|
||
self,
|
||
text: str,
|
||
config: HumanizeConfig,
|
||
) -> int:
|
||
"""[AC-MARH-11] 计算拟人化 delay。"""
|
||
text_length = len(text)
|
||
|
||
for bucket in self._length_buckets:
|
||
if bucket.min_length <= text_length < bucket.max_length:
|
||
delay = bucket.delay_ms
|
||
return max(config.min_delay_ms, min(delay, config.max_delay_ms))
|
||
|
||
return config.min_delay_ms
|
||
|
||
def get_config(self) -> HumanizeConfig:
|
||
"""获取当前配置。"""
|
||
return self._config
|
||
|
||
|
||
_segment_humanizer: SegmentHumanizer | None = None
|
||
|
||
|
||
def get_segment_humanizer(
|
||
config: HumanizeConfig | None = None,
|
||
) -> SegmentHumanizer:
|
||
"""获取或创建 SegmentHumanizer 实例。"""
|
||
global _segment_humanizer
|
||
if _segment_humanizer is None:
|
||
_segment_humanizer = SegmentHumanizer(config=config)
|
||
return _segment_humanizer
|