Misaki
587 lines
29 KiB
Python
587 lines
29 KiB
Python
# server_core/core.py
|
|
|
|
"""
|
|
统一业务，对外提供启动接口
|
|
业务数据流向：
|
|
Yosuga[User Audio Info Struct] ->(WebSocket) Yosuga_server[asr_module] -> Text
|
|
Yosuga_server[Come from Yosuga Audio ASR Text] ->(Func call) Yosuga_server[llm_core] -> Ins and Text
|
|
|
|
Yosuga_server[Come from llm_core Text]->(WebSocket) Yosuga
|
|
Yosuga_server[Come from llm_core Text]->(Func call) Yosuga_server[TTS] -> Audio Data
|
|
Yosuga_server[Audio Data] ->(WebSocket) Yosuga
|
|
|
|
Yosuga_embedded[Devices Control Info] ->(WebSocket) Yosuga_server[embedded_core]-> Ins
|
|
Yosuga_embedded[Devices Control Info] ->(Serial) Yosuga[SerialManager] -> ForWord To Yosuga_server
|
|
Yosuga[Come from embedded Info] ->(WebSocket) Yosuga_server[llm_core] -> Ins
|
|
|
|
UI_TARS[Mind and x&y Info] ->(Func call) Yosuga_server[llm_core] -> Ins
|
|
Yosuga_server[Come from UI_TARS Ins] ->(WebSocket) Yosuga
|
|
|
|
Yosuga[Live2D Control Info] ->(WebSocket) Yosuga_server[llm_core] -> Ins
|
|
Yosuga_server[Live2D Control Ins] ->(Websocket) Yosuga
|
|
|
|
Yosuga_server[agent memory] ->(Func call) Yosuga_server[Memory Uint]
|
|
"""
|
|
import asyncio
|
|
from typing import Optional, List, Dict, Any
|
|
from loguru import logger
|
|
import json
|
|
from src.modules.websocket_base_module.dto.third_dtos import (
|
|
AudioDataDTO, AudioDataTransferObject,
|
|
ScreenShotDataDTO, ScreenShotDataTransferObject
|
|
)
|
|
from src.modules.websocket_base_module.dto.second_dtos import JsonDTO, get_json_dto_instance
|
|
from src.modules.websocket_base_module.websocket_core.core_ws_server import WebSocketServer, get_ws_server
|
|
|
|
from src.modules.device_control_module.device_control_core.ui_tars_.ui_tars_client import UITarsClient, UITarsClientConfig
|
|
|
|
from src.modules.asr_module.client.asr_client import create_asr_client, ASRClientConfig, ASRClientAsync
|
|
|
|
from src.modules.tts_module.tts_core.gpt_sovits.gpt_sovits_client import StreamingMode, TTSConfig, GPTSoVITSClient
|
|
|
|
from src.server_core.llm_core.llm_core import (
|
|
LLMCoreConfig, ModelConfig,
|
|
YosugaLLMCore, ModelProvider,
|
|
LLMCoreAnalysisBase,
|
|
YosugaAudioResponseData, YosugaUITARSResponseData,
|
|
YosugaUITARSRequestData, YosugaEmbeddedResponseData
|
|
)
|
|
|
|
from src.server_core.yosuga_embedded_server import (
|
|
YosugaServer, ServerConfig
|
|
)
|
|
from src.server_core.yosuga_embedded_server.device_dto import DeviceDataDTO
|
|
from src.server_core.llm_core.llm_core_prompt_manager import YosugaEmbedded
|
|
|
|
from src.modules.websocket_base_module.dto.dto_templates.auto_agent_data_dto import AutoAgentDataTransferObject
|
|
from src.config.config import cfg
|
|
|
|
|
|
class YosugaServerCore:
|
|
"""
|
|
异步单例类
|
|
"""
|
|
|
|
_instance: Optional["YosugaServerCore"] = None
|
|
_lock = asyncio.Lock()
|
|
|
|
# 组合必要的工具类
|
|
ws_server: WebSocketServer
|
|
json_dto: JsonDTO
|
|
audio_dto: AudioDataDTO
|
|
screenshot_dto: ScreenShotDataDTO
|
|
|
|
asr_client: ASRClientAsync # 异步asr client
|
|
tts_client: GPTSoVITSClient # tts client
|
|
auto_agent_client: UITarsClient # GUI自动化agent
|
|
|
|
llm_core: YosugaLLMCore = None # llm core
|
|
|
|
embedded_server: YosugaServer # 嵌入式设备管理框架
|
|
device_dto: DeviceDataDTO # 设备数据分发器
|
|
|
|
# @classmethod
|
|
# async def get_instance(cls) -> "YosugaServerCore":
|
|
# """异步单例工厂"""
|
|
# if cls._instance is None:
|
|
# async with cls._lock:
|
|
# if cls._instance is None:
|
|
# logger.info("Initializing YosugaServerCore...")
|
|
# # 创建实例
|
|
# instance = cls.__new__(cls)
|
|
#
|
|
# # 按依赖顺序初始化数据分发器
|
|
# instance.ws_server = await get_ws_server()
|
|
# instance.json_dto = await get_json_dto_instance(instance.ws_server)
|
|
# instance.audio_dto = AudioDataDTO(instance.json_dto) # 音频分发器
|
|
# instance.audio_dto.register_audio_callback(instance._handle_audio_data) # 注册音频处理函数
|
|
# instance.screenshot_dto = ScreenShotDataDTO(instance.json_dto) # 截图分发器
|
|
# instance.screenshot_dto.register_screenshot_callback(instance._handle_screenshot_data) # 注册截图处理函数
|
|
#
|
|
# instance.asr_client = create_asr_client(use_async=True, base_url=cfg.asr.url)
|
|
# instance.tts_client = GPTSoVITSClient(host=cfg.tts.host, port=cfg.tts.port, debug=True)
|
|
# # 切换GPT_SoVITS模型
|
|
# await instance.tts_client.set_gpt_weights(cfg.tts.gpt_model_name)
|
|
# await instance.tts_client.set_sovits_weights(cfg.tts.sovits_model_name)
|
|
#
|
|
# instance.auto_agent_client = UITarsClient(UITarsClientConfig(
|
|
# deployment_type=cfg.auto_agent.deployment_type,
|
|
# base_url=cfg.auto_agent.base_url,
|
|
# model_name=cfg.auto_agent.model_name,
|
|
# temperature=cfg.auto_agent.temperature,
|
|
# max_tokens=cfg.auto_agent.max_tokens
|
|
# ))
|
|
#
|
|
# instance.llm_core = YosugaLLMCore(
|
|
# model_config=ModelConfig( # TODO 同上
|
|
# provider=ModelProvider.OPENAI,
|
|
# model_name=cfg.ai.model_name,
|
|
# base_url=cfg.ai.base_url,
|
|
# api_key=cfg.ai.api_key,
|
|
# temperature=cfg.ai.temperature,
|
|
# max_tokens=cfg.ai.max_tokens
|
|
# ),
|
|
# core_config=LLMCoreConfig( # TODO 同上
|
|
# max_context_tokens=cfg.llm_core.max_context_tokens,
|
|
# enable_history=cfg.llm_core.enable_history,
|
|
# role_setting=cfg.llm_core.role_character,
|
|
# language=cfg.llm_core.language, # 回复使用语言
|
|
# auto_dispatch=True,
|
|
# dispatch_async=True # 启用异步分发
|
|
# )
|
|
# )
|
|
# instance.register_llm_core_analysis() # 注册解析器
|
|
# instance.register_llm_core_action() # 注册分发器
|
|
# instance.llm_core.register_overflow_handler(instance._handle_overflow_logger) # 注册上下文溢出处理回调
|
|
#
|
|
# cls._instance = instance
|
|
# logger.success("YosugaServerCore initialized")
|
|
# return cls._instance
|
|
@classmethod
|
|
async def get_instance(cls) -> "YosugaServerCore":
|
|
"""异步单例工厂"""
|
|
if cls._instance is None:
|
|
async with cls._lock:
|
|
if cls._instance is None:
|
|
logger.info("Initializing YosugaServerCore...")
|
|
|
|
# 强制初始化配置
|
|
from src.config.config import _ensure_initialized
|
|
from dataclasses import asdict, is_dataclass
|
|
|
|
real_cfg = _ensure_initialized()
|
|
|
|
# 辅助函数：递归转换为 dict
|
|
def to_dict(obj):
|
|
if isinstance(obj, dict):
|
|
return obj
|
|
if is_dataclass(obj) and not isinstance(obj, type):
|
|
return asdict(obj)
|
|
return {}
|
|
|
|
# 提取各个配置段并转换为 dict（关键修复）
|
|
cfg_dict = {
|
|
'ai': to_dict(getattr(real_cfg, 'ai', {})),
|
|
'tts': to_dict(getattr(real_cfg, 'tts', {})),
|
|
'asr': to_dict(getattr(real_cfg, 'asr', {})),
|
|
'auto_agent': to_dict(getattr(real_cfg, 'auto_agent', {})),
|
|
'llm_core': to_dict(getattr(real_cfg, 'llm_core', {})),
|
|
}
|
|
|
|
logger.debug(f"配置提取完成: ai={type(cfg_dict['ai'])}, tts={type(cfg_dict['tts'])}")
|
|
|
|
# 创建实例
|
|
instance = cls.__new__(cls)
|
|
|
|
# 按依赖顺序初始化数据分发器
|
|
instance.ws_server = await get_ws_server()
|
|
instance.json_dto = await get_json_dto_instance(instance.ws_server)
|
|
instance.audio_dto = AudioDataDTO(instance.json_dto)
|
|
instance.audio_dto.register_audio_callback(instance._handle_audio_data)
|
|
instance.screenshot_dto = ScreenShotDataDTO(instance.json_dto)
|
|
instance.screenshot_dto.register_screenshot_callback(instance._handle_screenshot_data)
|
|
|
|
# ASR 客户端
|
|
asr_cfg = cfg_dict.get('asr', {})
|
|
instance.asr_client = create_asr_client(
|
|
use_async=True,
|
|
base_url=asr_cfg.get('url', 'http://localhost:20260/')
|
|
)
|
|
|
|
# TTS 客户端
|
|
tts_cfg = cfg_dict.get('tts', {})
|
|
instance.tts_client = GPTSoVITSClient(
|
|
host=tts_cfg.get('host', 'localhost'),
|
|
port=tts_cfg.get('port', 20261),
|
|
debug=True
|
|
)
|
|
|
|
# 切换 GPT_SoVITS 模型
|
|
# await instance.tts_client.set_gpt_weights(
|
|
# tts_cfg.get('gpt_model_name', 'GPT_weights_v2Pro/Yosuga_Airi-e32.ckpt')
|
|
# )
|
|
# await instance.tts_client.set_sovits_weights(
|
|
# tts_cfg.get('sovits_model_name', 'SoVITS_weights_v2Pro/Yosuga_Airi_e16_s864.pth')
|
|
# )
|
|
|
|
# Auto Agent 客户端
|
|
auto_cfg = cfg_dict.get('auto_agent', {})
|
|
instance.auto_agent_client = UITarsClient(UITarsClientConfig(
|
|
deployment_type=auto_cfg.get('deployment_type', 'lmstudio'),
|
|
base_url=auto_cfg.get('base_url', 'http://localhost:1234/v1'),
|
|
model_name=auto_cfg.get('model_name', 'ui-tars-1.5-7b@q4_k_m'),
|
|
temperature=auto_cfg.get('temperature', 0.1),
|
|
max_tokens=auto_cfg.get('max_tokens', 16384)
|
|
))
|
|
|
|
# LLM Core
|
|
ai_cfg = cfg_dict.get('ai', {})
|
|
llm_cfg = cfg_dict.get('llm_core', {})
|
|
|
|
instance.llm_core = YosugaLLMCore(
|
|
model_config=ModelConfig(
|
|
provider=ModelProvider.OPENAI,
|
|
model_name=ai_cfg.get('model_name', 'qwen/qwen3-4b-2507'),
|
|
base_url=ai_cfg.get('base_url', 'http://localhost:1234/v1'),
|
|
api_key=ai_cfg.get('api_key'),
|
|
temperature=ai_cfg.get('temperature', 0.4),
|
|
max_tokens=ai_cfg.get('max_tokens', 8192)
|
|
),
|
|
core_config=LLMCoreConfig(
|
|
max_context_tokens=llm_cfg.get('max_context_tokens', 2048),
|
|
enable_history=llm_cfg.get('enable_history', True),
|
|
role_setting=llm_cfg.get('role_character',
|
|
'你是由Misakiotoha开发的助手稲葉愛理ちゃん，可以和用户一起玩游戏，聊天，做各种事情，性格抽象，没事爱整整活。'),
|
|
language=llm_cfg.get('language', '中文'),
|
|
auto_dispatch=True,
|
|
dispatch_async=True
|
|
)
|
|
)
|
|
|
|
# 注册 YosugaEmbedded 提示词模块
|
|
instance.llm_core.register_prompt_module(YosugaEmbedded())
|
|
logger.info("[Core] 嵌入式设备提示词模块已注册")
|
|
|
|
# 初始化嵌入式设备管理框架
|
|
instance.embedded_server = YosugaServer(
|
|
config=ServerConfig(
|
|
device_conflict_strategy="rename",
|
|
max_concurrent_calls=10,
|
|
device_timeout=30.0,
|
|
)
|
|
)
|
|
instance.device_dto = DeviceDataDTO(
|
|
instance.json_dto, instance.embedded_server
|
|
)
|
|
# 当 YosugaServer 需要发送 RPC 到设备时，通过 WebSocket 发出 device_command
|
|
instance.embedded_server.on_device_message = (
|
|
instance._on_device_message
|
|
)
|
|
# 当设备能力变更时，更新 LLM 系统提示词中的状态表
|
|
instance.embedded_server.on_capabilities_changed = (
|
|
instance._on_capabilities_changed
|
|
)
|
|
logger.success("[Core] 嵌入式设备管理框架已初始化")
|
|
|
|
# 注册设备 RPC 响应回调（设备结果回来后喂回 LLM）
|
|
instance.device_dto.register_device_callback(
|
|
instance._on_device_rpc_response
|
|
)
|
|
instance._pending_rpc: Optional[dict] = None
|
|
|
|
instance.register_llm_core_analysis()
|
|
instance.register_llm_core_action()
|
|
instance.llm_core.register_overflow_handler(instance._handle_overflow_logger)
|
|
|
|
cls._instance = instance
|
|
logger.success("YosugaServerCore initialized")
|
|
|
|
return cls._instance
|
|
|
|
|
|
def register_llm_core_action(self):
|
|
"""
|
|
注册llm_core的分发器
|
|
"""
|
|
if self.llm_core is None:
|
|
raise Exception("LLMCore is not initialized")
|
|
self.llm_core.register_action_handler("audio_text", self._handle_audio_response, is_async=True)
|
|
self.llm_core.register_action_handler("auto_agent", self._handle_auto_agent, is_async=True)
|
|
self.llm_core.register_action_handler("call_auto_agent", self._handle_call_auto_agent, is_async=True)
|
|
self.llm_core.register_action_handler("embedded_control", self._handle_embedded_control, is_async=True)
|
|
self.llm_core.set_fallback_handler(self._handle_fallback)
|
|
|
|
def register_llm_core_analysis(self):
|
|
"""
|
|
注册llm_core的输出解析器
|
|
"""
|
|
if self.llm_core is None:
|
|
raise Exception("LLMCore is not initialized")
|
|
self.llm_core.register_analysis_model(YosugaAudioResponseData)
|
|
self.llm_core.register_analysis_model(YosugaUITARSResponseData)
|
|
self.llm_core.register_analysis_model(YosugaUITARSRequestData)
|
|
self.llm_core.register_analysis_model(YosugaEmbeddedResponseData)
|
|
|
|
def _handle_overflow_logger(self, history: List[Any], metadata: Dict[str, Any]):
|
|
"""上下文溢出记录，仅打印日志"""
|
|
print(f" 上下文溢出！")
|
|
print(f" 模型: {metadata['model']}")
|
|
print(f" 消息数: {metadata['message_count']}")
|
|
print(f" Token: {metadata['estimated_tokens']}/{metadata['limit']}")
|
|
print(f" 即将遗忘 {len(history) // 2} 条旧消息")
|
|
|
|
async def _handle_audio_data(self, audio_data: AudioDataTransferObject):
|
|
"""
|
|
音频数据接收call back
|
|
Yosuga_server只有接受到每次这个audio数据才会跑一次
|
|
"""
|
|
logger.info("Received audio data")
|
|
# 在此处客户端发送的音频数据必定不是流式数据(考虑客户端发送数据给服务端往往是在本地的，速度极快)
|
|
# 将音频数据发送给asr转换成文本信息，音频数据格式为wav
|
|
# TODO: 考虑在此处做一个简单的vad检测，如果客户端发送的音频是静音的，则不把请求发给llm_core
|
|
asr_response = await self.asr_client.transcribe_bytes(audio_data.data)
|
|
if not asr_response.success:
|
|
logger.error(f"ASR failed: {asr_response.error}")
|
|
asr_result = asr_response.data # 获取asr结果
|
|
# 将asr结果发送给llm_core进行处理
|
|
llm_result = await self.llm_core.interact(
|
|
user_input={ # 构造用户输入信息
|
|
"text": asr_result.text,
|
|
"confidence": asr_result.confidence
|
|
}
|
|
) # llm_core会自动进行处理并通过执行器异步返回各种相关的数据
|
|
|
|
async def _handle_screenshot_data(self, screenshot_data: ScreenShotDataTransferObject):
|
|
"""
|
|
屏幕截图数据接收call back
|
|
将llm_core的回复封装后提交给auto_agent模块，获得自动化agent的返回之后再返回给llm_core
|
|
"""
|
|
logger.info(f"Received screenshot data {len(screenshot_data.RealTimeScreenShot)}")
|
|
if not screenshot_data.isSuccess: # 如果客户端截图失败
|
|
logger.error("Screenshot failed")
|
|
return # 直接提前结束回调，不向llm_core发送结果
|
|
# TODO 对于设备描述信息(screenshot_data.DescribeInfo)，考虑加入到auto_agent的输入中，增强识别准确率
|
|
# 构造请求 异步调用
|
|
logger.debug(f"screenshot_data.LLMResponse(来自llm_core向auto_agent的输入): {screenshot_data.LLMResponse}")
|
|
logger.debug(f"客户端设备信息: {screenshot_data.DescribeInfo}")
|
|
auto_agent_response: str = await self.auto_agent_client.call_async(screenshot_data.LLMResponse,
|
|
screenshot_data.RealTimeScreenShot)
|
|
logger.debug(f"auto_agent_response(auto_agent原生返回结果): {auto_agent_response}")
|
|
# 将auto_agent的返回结果发送给llm_core
|
|
await self.llm_core.interact(
|
|
user_input={ # 构造auto_agent输入信息
|
|
"auto_agent": auto_agent_response
|
|
}
|
|
)
|
|
|
|
async def _handle_audio_response(self, data: YosugaAudioResponseData):
|
|
"""
|
|
llm_core异步处理器：语音回复
|
|
将llm_core的回复封装后提交给tts模块，调用tts模块中的流式返回，并将流式frame返回给Yosuga客户端
|
|
"""
|
|
if data.type == "audio_text":
|
|
logger.info("Handling audio response")
|
|
try:
|
|
# 使用最快模式流式输出
|
|
chunk_count = 0
|
|
# async for chunk in await self.tts_client.tts(
|
|
# text=data.response_text,
|
|
# ref_audio_path="uploaded_audio/test_voice.wav", # TODO 需要替换成config或者后续设计情感系统
|
|
# text_lang="ja",
|
|
# prompt_lang="ja",
|
|
# prompt_text="もう!こんなところで何やってるんだよ!", # 参考语音的真实文本
|
|
# streaming_mode=StreamingMode.FASTEST, # 模式3：快速流式
|
|
# media_type="wav"
|
|
# ):
|
|
async for chunk in await self.tts_client.tts(
|
|
text=data.response_text,
|
|
ref_audio_path="uploaded_audio/kq.wav", # TODO 需要替换成config或者后续设计情感系统
|
|
text_lang="zh",
|
|
prompt_lang="zh",
|
|
prompt_text="电闪雷鸣虽然有点吓人，但璃月港的防雷防火工事是一流的，不用担心。", # 参考语音的真实文本
|
|
streaming_mode=StreamingMode.FASTEST, # 模式3：快速流式
|
|
media_type="wav"
|
|
):
|
|
chunk_count += 1
|
|
# print(f"🎵 收到音频块 #{chunk_count}: {len(chunk.audio_data)} bytes")
|
|
if chunk_count == 1: # 如果是第一个音频块
|
|
# 构造音频首包发送给客户端
|
|
await self.audio_dto.send_audio_data(
|
|
AudioDataTransferObject(
|
|
data=chunk.audio_data,
|
|
isStream=True,
|
|
isStart=True,
|
|
sequence=chunk_count,
|
|
isEnd=False,
|
|
text=data.response_text
|
|
)
|
|
)
|
|
else: # 如果不是第一个音频块，则发送中间包给客户端
|
|
await self.audio_dto.send_audio_data(
|
|
AudioDataTransferObject(
|
|
data=chunk.audio_data,
|
|
isStream=True,
|
|
isStart=False,
|
|
sequence=chunk_count,
|
|
isEnd=False,
|
|
text=data.response_text
|
|
)
|
|
)
|
|
print(f"流式TTS完成！共{chunk_count}个音频块")
|
|
# 构造音频尾包发送给客户端(虚假的音频数据)
|
|
await self.audio_dto.send_audio_data(
|
|
AudioDataTransferObject(
|
|
data=b"0",
|
|
isStream=True,
|
|
isStart=False,
|
|
sequence=chunk_count + 1,
|
|
isEnd=True,
|
|
text=data.response_text
|
|
)
|
|
)
|
|
except Exception as e:
|
|
print(f"流式错误: {e}")
|
|
return {"status": "success", "executed": data.response_text}
|
|
return None
|
|
|
|
async def _handle_auto_agent(self, data: YosugaUITARSResponseData):
|
|
"""
|
|
llm_core异步处理器：处理自动化操作
|
|
将llm_core的回复封装后提交给Yosuga客户端，由客户端进行执行相关的GUI自动化操作
|
|
"""
|
|
# 构造并发送回复数据
|
|
await self.json_dto.send_json(
|
|
AutoAgentDataTransferObject.from_json(data.to_dict()).to_json()
|
|
)
|
|
return {"status": "success", "executed": data.Action}
|
|
|
|
async def _handle_call_auto_agent(self, data: YosugaUITARSRequestData):
|
|
"""
|
|
llm_core异步处理器：处理llm_core调用auto_agent需求
|
|
向客户端请求当前界面的截图，请求成功后由_handle_screenshot_data函数完成剩下的任务
|
|
"""
|
|
if data.type == "call_auto_agent":
|
|
logger.info("LLM Calling auto agent")
|
|
# 向客户端请求当前界面的截图的base64编码 加入llm回复的信息到截图请求DTO当中 方便_handle_screenshot_data构造请求
|
|
await self.screenshot_dto.send_screenshot_data(ScreenShotDataTransferObject(LLMResponse=data.llm_translation))
|
|
return {"status": "success", "executed": data.type}
|
|
|
|
async def _handle_embedded_control(self, data: YosugaEmbeddedResponseData):
|
|
"""
|
|
llm_core异步处理器：嵌入式设备控制
|
|
将LLM输出的 JSON-RPC 调用列表交由 YosugaServer 框架处理并路由到对应设备
|
|
"""
|
|
logger.info(f"Handling embedded control: {len(data.calls)} calls")
|
|
|
|
results = self.embedded_server.process_ai_response(json.dumps(data.calls))
|
|
logger.info(f"Embedded control results: {results}")
|
|
|
|
# 保存 pending RPC 信息，等设备异步响应回来后喂回 LLM
|
|
if results and len(results) > 0:
|
|
first_call = results[0]
|
|
self._pending_rpc = {
|
|
"device_id": first_call.get("device_id"),
|
|
"method": first_call.get("method"),
|
|
"call_id": first_call.get("id"),
|
|
"original_response_text": data.response_text or "",
|
|
}
|
|
|
|
# 如果 LLM 同时返回了需要回复用户的文本，通过 TTS 播报
|
|
if data.response_text:
|
|
try:
|
|
chunk_count = 0
|
|
# async for chunk in await self.tts_client.tts(
|
|
# text=data.response_text,
|
|
# ref_audio_path="uploaded_audio/test_voice.wav",
|
|
# text_lang="ja",
|
|
# prompt_lang="ja",
|
|
# prompt_text="もう!こんなところで何やってるんだよ!",
|
|
# streaming_mode=StreamingMode.FASTEST,
|
|
# media_type="wav"
|
|
# ):
|
|
async for chunk in await self.tts_client.tts(
|
|
text=data.response_text,
|
|
ref_audio_path="uploaded_audio/kq.wav", # TODO 需要替换成config或者后续设计情感系统
|
|
text_lang="zh",
|
|
prompt_lang="zh",
|
|
prompt_text="电闪雷鸣虽然有点吓人，但璃月港的防雷防火工事是一流的，不用担心。", # 参考语音的真实文本
|
|
streaming_mode=StreamingMode.FASTEST, # 模式3：快速流式
|
|
media_type="wav"
|
|
):
|
|
chunk_count += 1
|
|
if chunk_count == 1:
|
|
await self.audio_dto.send_audio_data(
|
|
AudioDataTransferObject(
|
|
data=chunk.audio_data,
|
|
isStream=True, isStart=True,
|
|
sequence=chunk_count, isEnd=False,
|
|
text=data.response_text
|
|
)
|
|
)
|
|
else:
|
|
await self.audio_dto.send_audio_data(
|
|
AudioDataTransferObject(
|
|
data=chunk.audio_data,
|
|
isStream=True, isStart=False,
|
|
sequence=chunk_count, isEnd=False,
|
|
text=data.response_text
|
|
)
|
|
)
|
|
await self.audio_dto.send_audio_data(
|
|
AudioDataTransferObject(
|
|
data=b"0",
|
|
isStream=True, isStart=False,
|
|
sequence=chunk_count + 1, isEnd=True,
|
|
text=data.response_text
|
|
)
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Embedded control TTS error: {e}")
|
|
|
|
return {"status": "success", "calls": len(data.calls)}
|
|
|
|
def _on_device_rpc_response(self, device_id: str, payload: dict):
|
|
"""DeviceDataDTO 回调：设备 RPC 响应回来时触发，喂回 LLM"""
|
|
if self._pending_rpc and self._pending_rpc.get("device_id") == device_id:
|
|
call_id = payload.get("id")
|
|
if call_id is None or call_id == self._pending_rpc.get("call_id"):
|
|
pending = self._pending_rpc
|
|
self._pending_rpc = None
|
|
asyncio.create_task(self._continue_with_device_result(device_id, payload, pending))
|
|
|
|
async def _continue_with_device_result(self, device_id: str, payload: dict, pending: dict):
|
|
"""设备 RPC 结果回来后，喂回 LLM 生成最终回复并 TTS"""
|
|
method = pending.get("method", "unknown")
|
|
original_text = pending.get("original_response_text", "")
|
|
|
|
result_str = json.dumps(payload.get("result", payload), ensure_ascii=False)
|
|
followup_input = (
|
|
f"你之前请求设备 {device_id} 执行了 {method} 操作，"
|
|
f"现在设备返回了结果：{result_str}。\n"
|
|
f"你之前的回复是：'{original_text}'\n"
|
|
f"请基于设备返回的实际结果，用自然语言重新组织回复，告诉用户结果。"
|
|
)
|
|
|
|
try:
|
|
llm_result = await self.llm_core.interact(user_input={"text": followup_input})
|
|
logger.info(f"[Core] 设备结果回送 LLM 完成: {llm_result}")
|
|
except Exception as e:
|
|
logger.error(f"[Core] 设备结果回送 LLM 失败: {e}")
|
|
|
|
def _on_device_message(self, device_id: str, rpc_call: str) -> Optional[str]:
|
|
"""YosugaServer 的设备消息回调：通过 WebSocket 发送 RPC 到客户端"""
|
|
logger.info(f"[Core] 发送设备命令到 {device_id}")
|
|
asyncio.create_task(self.device_dto.send_device_command(device_id, rpc_call))
|
|
return None
|
|
|
|
def _on_capabilities_changed(self, capabilities: dict):
|
|
"""设备能力变更回调：更新 LLM 系统提示词中的状态表"""
|
|
functions_str = json.dumps(capabilities.get("functions", []), ensure_ascii=False, indent=2)
|
|
device_str = json.dumps(capabilities.get("devices", {}), ensure_ascii=False, indent=2)
|
|
state_table = (
|
|
f"【当前在线设备】\n{device_str}\n\n"
|
|
f"【设备可用函数】\n{functions_str}"
|
|
)
|
|
self.llm_core.core_config.system_state_table = state_table
|
|
logger.info(f"[Core] 系统状态表已更新 | 设备: {capabilities.get('device_count', 0)} 台 | 函数: {capabilities.get('function_count', 0)} 个")
|
|
|
|
def _handle_fallback(self, data: LLMCoreAnalysisBase):
|
|
"""
|
|
llm_core同步处理器：回退处理器
|
|
"""
|
|
logger.debug(f" [Fallback] 未知类型数据: {data.type}, 内容: {data.model_dump_json()}")
|
|
|
|
async def run(self):
|
|
"""启动服务器"""
|
|
logger.info("Yosuga Server Websocket Core 启动中...")
|
|
await self.ws_server.run(host="0.0.0.0")
|
|
|
|
|
|
# 使用方式
|
|
async def main():
|
|
core = await YosugaServerCore.get_instance()
|
|
await core.run()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(main()) |