diff --git a/3rdparty/autogui-cpp/src/Autogui.cpp b/3rdparty/autogui-cpp/src/Autogui.cpp index 1ea84e6..5506bc1 100644 --- a/3rdparty/autogui-cpp/src/Autogui.cpp +++ b/3rdparty/autogui-cpp/src/Autogui.cpp @@ -11,6 +11,10 @@ #include #include +#ifndef M_PI +# define M_PI 3.14159265358979323846 +#endif + namespace AutoGUI { // 内部辅助函数 diff --git a/src/Core/Inc/AppCore.h b/src/Core/Inc/AppCore.h index d86295c..cdaa186 100644 --- a/src/Core/Inc/AppCore.h +++ b/src/Core/Inc/AppCore.h @@ -8,4 +8,32 @@ * 客户端业务核心 * 1. 处理来自服务端的数据，分发并执行 * 2. 完成非阻塞的事件循环处理，构建业务状态机 - */ \ No newline at end of file + */ +#include +#include + +class AppCore final : public QObject { +Q_OBJECT +Q_DISABLE_COPY(AppCore) // 禁用拷贝 + +private: + /** + * 构造函数私有化 + * @param parent + */ + explicit AppCore(QObject *parent = nullptr); // 并不将本模块挂在对象树当中，因为本模块为单例类，内存自行管理 + + static QScopedPointer m_instance; // 单例类 + static QMutex m_mutex; +private slots: + // 业务接收槽函数 + +public: + // 单例访问点 + static AppCore *getInstance(); + // 显式销毁 + static void destroy(); + + ~AppCore() override; + +}; \ No newline at end of file diff --git a/src/Core/Inc/GLCore.h b/src/Core/Inc/GLCore.h index cec516b..17ca90e 100644 --- a/src/Core/Inc/GLCore.h +++ b/src/Core/Inc/GLCore.h @@ -3,6 +3,9 @@ #include #include #include "menu.h" +#ifdef Q_OS_WIN +#include +#endif class GLCore final : public QOpenGLWidget { @@ -81,7 +84,6 @@ private: bool isRightPressed; /// 鼠标右键是否按下 QPoint currentPos; /// 当前鼠标位置 #ifdef Q_OS_WIN -#include private: HWND hwnd; // Windows窗口句柄 void setWindowTransparentForMouse(bool transparent); diff --git a/src/Core/Src/AppCore.cpp b/src/Core/Src/AppCore.cpp index e0e52e7..fe90184 100644 --- a/src/Core/Src/AppCore.cpp +++ b/src/Core/Src/AppCore.cpp @@ -2,4 +2,53 @@ // Created by misaki on 2026/1/24. // -#include "AppCore.h" \ No newline at end of file +#include "AppCore.h" +#include + +#include "AudioDataHandle.h" +#include "AutoAgentHandle.h" +#include "ScreenShotReqDataHandle.h" + +// 初始化静态成员 +QScopedPointer AppCore::m_instance; +QMutex AppCore::m_mutex; + +// 单例实现 (QScopedPointer + Mutex) +AppCore* AppCore::getInstance() +{ + if (m_instance.isNull()) { + QMutexLocker locker(&m_mutex); + if (m_instance.isNull()) { + // 使用 reset 创建实例，因为构造函数是私有的 + m_instance.reset(new AppCore()); + } + } + return m_instance.data(); +} + +void AppCore::destroy() +{ + QMutexLocker locker(&m_mutex); + if (!m_instance.isNull()) { + m_instance.reset(); // 这会触发析构函数 + } +} + +AppCore::AppCore(QObject *parent) : QObject(parent) +{ + // 初始化业务解析单例 + AudioDataHandle::getInstance(); + AutoAgentHandle::getInstance(); + ScreenShotReqDataHandle::getInstance(); +} + +AppCore::~AppCore() +{ + // 析构业务解析单例 + ScreenShotReqDataHandle::destroy(); + AutoAgentHandle::destroy(); // 显式销毁 + AudioDataHandle::destroy(); + + + qDebug() << "AppCore destroyed"; +} diff --git a/src/DAO/Inc/NetWorkDO.h b/src/DAO/Inc/NetWorkDO.h index 56fd64c..46e958a 100644 --- a/src/DAO/Inc/NetWorkDO.h +++ b/src/DAO/Inc/NetWorkDO.h @@ -28,6 +28,7 @@ #include "DataTransferObjectBase.h" #include "AudioDataTransferObject.h" #include "AutoAgentDataObject.h" +#include "ScreenShotDataTransferObject.h" /** * NetworkDO */ @@ -54,8 +55,9 @@ public: signals: // 业务接收信号 - void audioPacketReceived(const AudioDataTransferObject& packet); // 音频数据准备完成信号 - void autoAgentPacketReceived(const AutoAgentDataObject& packet); // 自动代理数据包接收信号 + void audioPacketReceived(const AudioDataTransferObject& packet); // 音频数据准备完成信号 + void autoAgentPacketReceived(const AutoAgentDataObject& packet); // 自动代理数据包接收信号 + void screenShotPacketReceived(const ScreenShotDataTransferObject& packet); // 截图数据包接收信号 void errorOccurred(const QString& errorMsg); // 错误信号 diff --git a/src/DAO/Inc/ScreenShotDataTransferObject.h b/src/DAO/Inc/ScreenShotDataTransferObject.h new file mode 100644 index 0000000..875395b --- /dev/null +++ b/src/DAO/Inc/ScreenShotDataTransferObject.h @@ -0,0 +1,55 @@ +// +// Created by misaki on 2026/2/1. +// + +/** + * 数据传输对象 (DTO) 定义 + * ScreenShotDataTransferObject + * 与Yosuga_server中的是对等DTO + */ + +#pragma once +#include +#include +#include +#include +#include "DataTransferObjectBase.h" +// 前向声明，减少依赖 +class QJsonObject; +class ScreenShotDataTransferObject final : public DataTransferObjectBase{ +public: + // 构造函数（带默认值） + explicit ScreenShotDataTransferObject(QString owner = "client", + bool isSuccess = true, + QString realtimeScreenShot = "", + int width = 0, int height = 0, + QString describeInfo = "", QString LLMResponse = "" + ); + // 静态工厂方法 + static ScreenShotDataTransferObject fromJson(const QJsonObject& json); + + [[nodiscard]] QString type() const override { return "screenshot_req"; } + + // 序列化 + [[nodiscard]] QJsonObject toJson() const override; // 通过多态即可统一调用方式 + + // 链式调用设置 + ScreenShotDataTransferObject& setData(const QString& key, const QJsonValue& value) override; + + [[nodiscard]] QString owner() const { return m_owner; } + [[nodiscard]] bool isSuccess() const { return m_isSuccess; } + [[nodiscard]] QString realtimeScreenShot() const { return m_realtimeScreenShot; } + [[nodiscard]] int width() const { return m_width; } + [[nodiscard]] int height() const { return m_height; } + [[nodiscard]] QString describeInfo() const { return m_describeInfo; } + [[nodiscard]] QString LLMResponse() const { return m_LLMResponse; } + +private: + QString m_owner; /// 数据的拥有者(server or client) + bool m_isSuccess; /// 截图是否成功 + QString m_realtimeScreenShot; /// 客户端设备的实时截图数据(base64) + int m_width; /// 截图宽度 非必要字段 + int m_height; /// 截图高度 非必要字段 + QString m_describeInfo; /// 设备的描述信息(告知模型以做出更加准确的判断) 非必要字段 + QString m_LLMResponse; /// LLM的响应结果(由服务端发送时携带) +}; diff --git a/src/DAO/Src/NetWorkDO.cpp b/src/DAO/Src/NetWorkDO.cpp index 9846a0c..31274f7 100644 --- a/src/DAO/Src/NetWorkDO.cpp +++ b/src/DAO/Src/NetWorkDO.cpp @@ -65,13 +65,16 @@ void NetworkDO::onDataReceived(const QString& type, const QJsonObject& data) // 根据类型分发数据包 // 为什么分发做在这里，而不是统一数据再去分发，如果不在这里做分发通知，分开发信号，而使用统一的信号 // 如果有多个观察者，让观察者自动识别数据包，这会导致信号广播，容易引起性能问题(因为这里依赖的是Qt的信号与槽机制) - // TODO: 在此处使用工厂模式，根据type内容快速创建对应的对象 + // TODO: 考虑在此处使用工厂模式，根据type内容快速创建对应的对象 if (type == "audio_data") { emit audioPacketReceived(AudioDataTransferObject::fromJson(data)); // 构造并发送音频对象 } else if (type == "auto_agent") { emit autoAgentPacketReceived(AutoAgentDataObject::fromJson(data)); } + else if (type == "screenshot_req") { + emit screenShotPacketReceived(ScreenShotDataTransferObject::fromJson(data)); + } else { qWarning() << "[NetworkDO] Received unknown type:" << type; } diff --git a/src/DAO/Src/ScreenShotDataTransferObject.cpp b/src/DAO/Src/ScreenShotDataTransferObject.cpp new file mode 100644 index 0000000..07059c0 --- /dev/null +++ b/src/DAO/Src/ScreenShotDataTransferObject.cpp @@ -0,0 +1,75 @@ +// +// Created by misaki on 2026/2/1. +// +#include "ScreenShotDataTransferObject.h" +#include +#include +#include + +// 构造函数实现（初始化列表） +ScreenShotDataTransferObject::ScreenShotDataTransferObject(QString owner, + bool isSuccess, + QString realtimeScreenShot, + int width, int height, + QString describeInfo, QString LLMResponse) + : m_owner(std::move(owner)) + , m_isSuccess(isSuccess) + , m_realtimeScreenShot(std::move(realtimeScreenShot)) + , m_width(width) + , m_height(height) + , m_describeInfo(std::move(describeInfo)) + , m_LLMResponse(std::move(LLMResponse)) +{} + +// 静态工厂方法：从 JSON 反序列化 +ScreenShotDataTransferObject ScreenShotDataTransferObject::fromJson(const QJsonObject& json) { + // 逐个字段读取，不存在则用默认值 + const QString owner = json.value("Owner").toString("client"); + const bool isSuccess = json.value("isSuccess").toBool(false); + const QString realtimeScreenShot = json.value("RealTimeScreenShot").toString(); + const int width = json.value("Width").toInt(0); + const int height = json.value("Height").toInt(0); + const QString describeInfo = json.value("DescribeInfo").toString(); + const QString LLMResponse = json.value("LLMResponse").toString(); + + // 调用构造函数创建对象 + return ScreenShotDataTransferObject(owner, isSuccess, realtimeScreenShot, + width, height, describeInfo); +} + +// 序列化为 JSON +QJsonObject ScreenShotDataTransferObject::toJson() const { + QJsonObject json; + json["Owner"] = m_owner; + json["isSuccess"] = m_isSuccess; + json["RealTimeScreenShot"] = m_realtimeScreenShot; + json["Width"] = m_width; + json["Height"] = m_height; + json["DescribeInfo"] = m_describeInfo; + json["LLMResponse"] = m_LLMResponse; + return json; +} + +// 链式设置 +ScreenShotDataTransferObject& ScreenShotDataTransferObject::setData(const QString& key, + const QJsonValue& value) { + if (key == "Owner") { + m_owner = value.toString(); + } else if (key == "isSuccess") { + m_isSuccess = value.toBool(); + } else if (key == "RealTimeScreenShot") { + m_realtimeScreenShot = value.toString(); + } else if (key == "Width") { + m_width = value.toInt(); + } else if (key == "Height") { + m_height = value.toInt(); + } else if (key == "DescribeInfo") { + m_describeInfo = value.toString(); + } else if (key == "LLMResponse") { + m_LLMResponse = value.toString(); + } else { + qWarning() << "Unknown key:" << key << "for ScreenShotDataTransferObject"; + } + + return *this; // 返回自身引用，支持链式调用 +} \ No newline at end of file diff --git a/src/Handle/AudioHandle/Src/AudioInput.cpp b/src/Handle/AudioHandle/Src/AudioInput.cpp index 84f016b..b257208 100644 --- a/src/Handle/AudioHandle/Src/AudioInput.cpp +++ b/src/Handle/AudioHandle/Src/AudioInput.cpp @@ -53,7 +53,7 @@ void AudioInput::setAudioSettings(const int rate, const int channels) { m_format.setSampleRate(rate); m_format.setChannelCount(channels); - // 重要：Qt6 默认可能是 float，为了生成标准 WAV 且方便计算 RMS，强制设为 Int16 + // 为了生成标准 WAV 且方便计算 RMS，强制设为 Int16 m_format.setSampleFormat(QAudioFormat::Int16); // 检查设备是否支持该格式，不支持则使用最接近的 diff --git a/src/Handle/DataObjectHandle/Inc/AudioDataHandle.h b/src/Handle/DataObjectHandle/Inc/AudioDataHandle.h index d2be0c9..e10a6fe 100644 --- a/src/Handle/DataObjectHandle/Inc/AudioDataHandle.h +++ b/src/Handle/DataObjectHandle/Inc/AudioDataHandle.h @@ -13,7 +13,7 @@ class AudioDataHandle final : public QObject { Q_OBJECT Q_DISABLE_COPY(AudioDataHandle) // 禁用拷贝 - private: +private: /** * 构造函数私有化 * @param parent diff --git a/src/Handle/DataObjectHandle/Inc/ScreenShotReqDataHandle.h b/src/Handle/DataObjectHandle/Inc/ScreenShotReqDataHandle.h new file mode 100644 index 0000000..47ba083 --- /dev/null +++ b/src/Handle/DataObjectHandle/Inc/ScreenShotReqDataHandle.h @@ -0,0 +1,38 @@ +// +// Created by misaki on 2026/2/1. +// +#pragma once +#include +#include +#include "ScreenShotDataTransferObject.h" +class ScreenShotReqDataHandle final : public QObject +{ +Q_OBJECT +Q_DISABLE_COPY(ScreenShotReqDataHandle) // 禁用拷贝 + private: + /** + * 构造函数私有化 + * @param parent + */ + explicit ScreenShotReqDataHandle(QObject *parent = nullptr); // 并不将本模块挂在对象树当中，因为本模块为单例类，内存自行管理 + + static QScopedPointer m_instance; // 单例类 + static QMutex m_mutex; + +private slots: + // 业务接收槽函数，当获取到截图数据包时，进行解析并处理 + void onScreenShotPacketReceived(const ScreenShotDataTransferObject& packet) const; + + signals: + // 发送截图处理完成的信号，供界面显示使用 + void screenShotProcessed(const QPixmap& screenshot, const QString& description); +public: + // 单例访问点 + static ScreenShotReqDataHandle *getInstance(); + // 显式销毁 + static void destroy(); + + ~ScreenShotReqDataHandle() override; +private: + QString m_systemInfo; +}; \ No newline at end of file diff --git a/src/Handle/DataObjectHandle/Src/AudioDataHandle.cpp b/src/Handle/DataObjectHandle/Src/AudioDataHandle.cpp index feeb3c0..afcd026 100644 --- a/src/Handle/DataObjectHandle/Src/AudioDataHandle.cpp +++ b/src/Handle/DataObjectHandle/Src/AudioDataHandle.cpp @@ -42,15 +42,15 @@ AudioDataHandle::~AudioDataHandle() void AudioDataHandle::onAudioPacketReceived(const AudioDataTransferObject &packet) { // 管理并调用AudioOutput播放流式wav音频 - if (packet.isEnd()) { // 如果是结束包 + if (packet.isEnd()) { // 如果是结束包(空包) AudioOutput::getInstance()->stopStream(); // 停止播放 return; } - if (packet.isStart()) { // 如果是开始包 + if (packet.isStart()) { // 如果是开始包(单wav 44字节头) AudioOutput::getInstance()->startStream(packet.sampleRate(), packet.channelCount(), packet.bitDepth());; // 播放开始 return; } - // 否则播放即可 + // 否则加入播放队列即可 AudioOutput::getInstance()->pushStreamData(packet.audioData()); } diff --git a/src/Handle/DataObjectHandle/Src/ScreenShotReqDataHandle.cpp b/src/Handle/DataObjectHandle/Src/ScreenShotReqDataHandle.cpp new file mode 100644 index 0000000..856e40d --- /dev/null +++ b/src/Handle/DataObjectHandle/Src/ScreenShotReqDataHandle.cpp @@ -0,0 +1,66 @@ +// +// Created by misaki on 2026/2/1. +// +#include "ScreenShotReqDataHandle.h" +#include "NetWorkDO.h" +#include +#include +#include "ScreenShotDataTransferObject.h" +#include "ScreenHelperUtil.hpp" +// 初始化静态成员 +QScopedPointer ScreenShotReqDataHandle::m_instance; +QMutex ScreenShotReqDataHandle::m_mutex; + +// 单例实现 (QScopedPointer + Mutex) +ScreenShotReqDataHandle* ScreenShotReqDataHandle::getInstance() +{ + if (m_instance.isNull()) { + QMutexLocker locker(&m_mutex); + if (m_instance.isNull()) { + // 使用 reset 创建实例，因为构造函数是私有的 + m_instance.reset(new ScreenShotReqDataHandle()); + } + } + return m_instance.data(); +} + +void ScreenShotReqDataHandle::destroy() +{ + QMutexLocker locker(&m_mutex); + if (!m_instance.isNull()) { + m_instance.reset(); // 这会触发析构函数 + } +} + + +ScreenShotReqDataHandle::ScreenShotReqDataHandle(QObject *parent) : QObject(parent) +{ + connect(NetworkDO::getInstance(), &NetworkDO::screenShotPacketReceived, + this, &ScreenShotReqDataHandle::onScreenShotPacketReceived); + // 初始化时候就构造好关于当前运行平台的信息 + ScreenHelper::SystemInfo sysInfo = ScreenHelper::getSystemInfo(); + const QString sysText = QString("System: %1 OS Version: %2 Display Server: %3") + .arg(sysInfo.osType, sysInfo.osVersion, sysInfo.displayServer); + this->m_systemInfo = sysText; +} + +ScreenShotReqDataHandle::~ScreenShotReqDataHandle() +{ + qDebug() << "ScreenShotDataHandle destroyed"; +} + +void ScreenShotReqDataHandle::onScreenShotPacketReceived(const ScreenShotDataTransferObject &packet) const { + qDebug() << "ScreenShot packet request from:" << packet.owner(); + // 截图当前画面并构造对等DTO发送 + const ScreenHelper::ScreenshotResult result = ScreenHelper::captureFocusedScreen(); // 获取当前屏幕截图 + if (!result.success) { // 如果截图失败 + // TODO: 考虑失败时候构造一个错误DTO给服务端 + return; + } + ScreenShotDataTransferObject reback; // 构造返回的DTO + reback.setData("isSuccess", true).setData("RealTimeScreenShot", result.base64Data) + .setData("Width", result.width).setData("Height", result.height) + .setData("DescribeInfo", this->m_systemInfo); + // 发送DTO + NetworkDO::getInstance()->sendPacket(reback); +} diff --git a/src/UI/Setting/Src/NetworkPage.cpp b/src/UI/Setting/Src/NetworkPage.cpp index fe1953b..7a07d73 100644 --- a/src/UI/Setting/Src/NetworkPage.cpp +++ b/src/UI/Setting/Src/NetworkPage.cpp @@ -10,7 +10,6 @@ #include "websocketmanager.h" #include "NetWorkDO.h" #include -#include "AudioDataHandle.h" NetWorkPage::NetWorkPage(QWidget* parent) : BasePage(parent) @@ -73,7 +72,6 @@ void NetWorkPage::initUI() { void NetWorkPage::initWebSocketClient() { auto* client = WebSocketClient::getInstance(); // 获取单例实例(设置一个默认地址) auto* netDO = NetworkDO::getInstance(); - AudioDataHandle::getInstance(); // 初始化音频处理模块 // 注入：将底层发送能力赋予 NetworkDO netDO->registerSender([client](const QString& type, const QJsonObject& data){ client->sendJson(type, data); diff --git a/src/Utils/Inc/ScreenHelperUtil.hpp b/src/Utils/Inc/ScreenHelperUtil.hpp new file mode 100644 index 0000000..29b9970 --- /dev/null +++ b/src/Utils/Inc/ScreenHelperUtil.hpp @@ -0,0 +1,53 @@ +// +// Created by misaki on 2026/2/1. +// +/** + * 屏幕截图与系统信息获取工具类 + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +class ScreenHelper +{ +public: + // 系统信息struct + struct SystemInfo { + QString osType; // 例如: "windows", "linux", "macos" + QString osVersion; // 例如: "Windows 11 (10.0)", "Ubuntu 22.04" + QString displayServer; // 例如: "windows", "cocoa", "xcb" (X11), "wayland" + bool isWayland; // 专门标记是否为 Wayland + }; + + // 截图结果struct + struct ScreenshotResult { + bool success; // 是否成功 + QString base64Data; // 图片的Base64字符串 (PNG格式) + int width; // 图片宽度 + int height; // 图片高度 + QString screenName; // 屏幕名称 + QString errorMsg; // 如果失败，返回错误信息 + }; +public: + /** + * @brief 获取当前焦点屏幕的全屏截图并转换为Base64 \n + * 判定逻辑：优先取有焦点的窗口所在屏幕，若无，取鼠标所在屏幕 + */ + static ScreenshotResult captureFocusedScreen(); + + /** + * @brief 获取当前操作系统和显示服务信息 + */ + static SystemInfo getSystemInfo(); + +private: + // 私有构造，禁止实例化 + ScreenHelper() = default; +}; \ No newline at end of file diff --git a/src/Utils/Src/ScreenHelperUtil.cpp b/src/Utils/Src/ScreenHelperUtil.cpp new file mode 100644 index 0000000..cbc5b64 --- /dev/null +++ b/src/Utils/Src/ScreenHelperUtil.cpp @@ -0,0 +1,92 @@ +// +// Created by misaki on 2026/2/1. +// +#include +#include "ScreenHelperUtil.hpp" +ScreenHelper::ScreenshotResult ScreenHelper::captureFocusedScreen() +{ + ScreenHelper::ScreenshotResult result; + result.success = false; + // 获取目标屏幕 + QScreen *targetScreen = nullptr; + + // 首先尝试获取当前应用程序拥有焦点的窗口所在的屏幕 + QWindow *focusWindow = QGuiApplication::focusWindow(); + if (focusWindow) { + targetScreen = focusWindow->screen(); + } + // 如果没有窗口焦点或者窗口还没显示，获取鼠标光标所在的屏幕 + if (!targetScreen) { + targetScreen = QGuiApplication::screenAt(QCursor::pos()); + } + + // 如果以上都失败，回退到主屏幕 + if (!targetScreen) { + targetScreen = QGuiApplication::primaryScreen(); + } + + if (!targetScreen) { + result.errorMsg = "Critical Error: No detectible screen found."; + return result; + } + + // 获取屏幕基本信息 + result.screenName = targetScreen->name(); + + // 执行截图 + // grabWindow(0) 表示截取整个屏幕 + // 注：在 Wayland 上，这可能需要系统权限或会弹出确认框，或者在某些安全策略下返回黑色图像 + QPixmap pixmap = targetScreen->grabWindow(0); + + if (pixmap.isNull()) { + result.errorMsg = "Failed to grab screen content (Permission denied or System restriction)."; + return result; + } + + result.width = pixmap.width(); + result.height = pixmap.height(); + + // 转换为 Base64 + QByteArray byteArray; + QBuffer buffer(&byteArray); + buffer.open(QIODevice::WriteOnly); + // 保存为 PNG 格式，质量默认即可 + if (pixmap.save(&buffer, "PNG")) { + result.base64Data = QString::fromLatin1(byteArray.toBase64()); + result.success = true; + } else { + result.errorMsg = "Failed to encode image to PNG buffer."; + } + + return result; +} + +ScreenHelper::SystemInfo ScreenHelper::getSystemInfo() +{ + ScreenHelper::SystemInfo info; + // 获取操作系统类型 + info.osType = QSysInfo::productType(); + + // 获取详细版本 (例如 Windows 10/11, Ubuntu 20.04) + // prettyProductName() 通常能区分 Win10 和 Win11 + info.osVersion = QSysInfo::prettyProductName(); + + // 获取显示服务器类型 (Platform Plugin) + // 这里的返回值通常是 QPA 插件的名字 + // Windows -> "windows" + // macOS -> "cocoa" + // Linux X11 -> "xcb" + // Linux Wayland -> "wayland" + QString platformName = QGuiApplication::platformName(); + info.displayServer = platformName; + + // 专门判断 Wayland + info.isWayland = (platformName == "wayland"); + // 针对 Linux 做更细致的显示名称优化 + if (platformName == "xcb") { + info.displayServer = "X11 (xcb)"; + } else if (platformName == "wayland") { + info.displayServer = "Wayland"; + } + return info; +} \ No newline at end of file