From d4ae8443d8c2b5a5598ad8537b968cf8a982a729 Mon Sep 17 00:00:00 2001 From: Yuna-Celisse <29951517@qq.com> Date: Fri, 2 Oct 2026 21:40:29 +0800 Subject: [PATCH 1/2] feat(aichat): add bounded image input and visual follow-up support --- .github/workflows/code-quality.yml | 2 +- deploy/aichat-multimodal/Dockerfile | 6 + deploy/aichat-multimodal/README.md | 66 ++++++++ .../CommandReader.cs | 3 + .../OneBotMessageHandler.cs | 8 + .../Config/PluginConfig.cs | 4 + .../HuaJiBot.NET.Plugin.AIChat.csproj | 1 + src/HuaJiBot.NET.Plugin.AIChat/PluginMain.cs | 52 +++--- .../Service/VisionInput.cs | 133 +++++++++++++++ .../AIChatVisionIntegrationTest.cs | 106 ++++++++++++ src/HuaJiBot.NET.UnitTest/VisionInputTest.cs | 159 ++++++++++++++++++ src/HuaJiBot.NET/DataBase/MessageHistory.cs | 1 + .../Events/GroupMessageEventArgs.cs | 2 + 13 files changed, 520 insertions(+), 23 deletions(-) create mode 100644 deploy/aichat-multimodal/Dockerfile create mode 100644 deploy/aichat-multimodal/README.md create mode 100644 src/HuaJiBot.NET.Plugin.AIChat/Service/VisionInput.cs create mode 100644 src/HuaJiBot.NET.UnitTest/AIChatVisionIntegrationTest.cs create mode 100644 src/HuaJiBot.NET.UnitTest/VisionInputTest.cs diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 438b9b3..bb03925 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -16,7 +16,7 @@ jobs: dotnet-version: '10.x' - name: Run focused tests - run: dotnet test src/HuaJiBot.NET.UnitTest --filter 'FullyQualifiedName~DailySummaryTest|FullyQualifiedName~AsciiArtToolsTest|FullyQualifiedName~WebDataServiceTest' + run: dotnet test src/HuaJiBot.NET.UnitTest --filter 'FullyQualifiedName~DailySummaryTest|FullyQualifiedName~AsciiArtToolsTest|FullyQualifiedName~WebDataServiceTest|FullyQualifiedName~VisionInputTest' format-check: runs-on: ubuntu-latest diff --git a/deploy/aichat-multimodal/Dockerfile b/deploy/aichat-multimodal/Dockerfile new file mode 100644 index 0000000..ba5eecc --- /dev/null +++ b/deploy/aichat-multimodal/Dockerfile @@ -0,0 +1,6 @@ +# Build context contains linux-x64 CLI publish output and the Release AIChat DLL. +# Overlay the current production image to preserve its other plugins and MCP services. +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +COPY cli/ /app/bin/ +COPY aichat/HuaJiBot.NET.Plugin.AIChat.dll /app/plugins/HuaJiBot.NET.Plugin.AIChat.dll diff --git a/deploy/aichat-multimodal/README.md b/deploy/aichat-multimodal/README.md new file mode 100644 index 0000000..c0c7f21 --- /dev/null +++ b/deploy/aichat-multimodal/README.md @@ -0,0 +1,66 @@ +# AIChat image input + +The OneBot adapter exposes image attachments through `GroupMessageEventArgs.ImageUrls`. +Typed URLs remain text. AIChat downloads attachments and sends validated image bytes as +Chat Completions image blocks. This first implementation covers OneBot/QQ; other adapters +can populate the same optional event field. PR #65 and its KOOK adapter remain separate. + +Under `Plugins.AIChat`, set `SupportsVision: true` only for a model with verified image +input support. `agnes-3.0-flash` and `agnes-2.5-flash` document image understanding; +`agnes-image-2.5-flash` is an image generation endpoint, not a chat model. +Defaults: `MaxImagesPerRequest: 4`, `MaxImageBytes: 8388608` (per image). +Requests include referenced historical images in the count. Old history remains readable. +Visual requests use reconstructed reply history and short-lived agent sessions, so image +bytes do not accumulate in the per-group session cache. + +Users can send `@bot + image`, `@bot + image` (default image description), +or reply to a bot response with a follow-up question. Images from earlier AI exchanges +are restored from history. Replying to an arbitrary image that never reached AIChat is +not supported. Expired image URLs require resending the image. + +Only public HTTP(S) attachment addresses are downloaded. Redirects and private-network +destinations are rejected. Download time is capped at 20 seconds and the complete request +at 90 seconds. PNG, JPEG, WebP and GIF are supported. Downloads are held in memory; +history stores source URLs, not Base64. AI request logs report counts only, and SDK +content logging is disabled when vision is enabled. An unsupported model, failed image +download, invalid format or exceeded limit produces an explicit user-facing message. + +## Validation + +```sh +dotnet test src/HuaJiBot.NET.UnitTest --filter FullyQualifiedName~VisionInputTest +``` + +Tests cover OneBot parsing, real image content and OpenAI-compatible serialization, +text-only behavior, capability gating, historical image limits, legacy history, invalid +URLs, network address checks, download failure and content validation. Real acceptance +requires a non-bot account to send a picture and mention in the configured QQ group, +then confirm the response describes details visible only in the picture and that a +reply-based follow-up retains the image context. + +## Production overlay deployment + +Following Issue #38, stop the bot before changing configuration. Back up `compose.yaml`, +`config.json` and plugin data; tag the current image for rollback. Do not restart other +Compose services. Build a Linux CLI and the Release AIChat plugin from the same commit: + +```sh +dotnet publish src/HuaJiBot.NET.CLI -c Release -f net10.0 -r linux-x64 \ + --no-self-contained -p:PublishSingleFile=true -o /tmp/huajibot-vision/cli +dotnet build src/HuaJiBot.NET.Plugin.AIChat -c Release -o /tmp/huajibot-vision/aichat +docker build --build-arg BASE_IMAGE= \ + -f deploy/aichat-multimodal/Dockerfile -t /tmp/huajibot-vision +``` + +The overlay assumes production already includes this branch's AIChat dependencies +(Microsoft.Agents.AI 1.1.0 and Microsoft.Extensions.AI.OpenAI 10.4.1). Verify versions +before using it; if dependencies differ, build the full image or update dependencies +explicitly. It preserves the current PushPanel and other plugin binaries. + +With the bot stopped, enable vision, change only its Compose image and start: +`docker compose up -d --no-deps huaji-bot-dotnet`. Verify startup, OneBot connection, +plugin loading, real image response, historical follow-up, text-only response and +existing panel behavior. Compare configuration against backup excluding the new vision +fields. Roll back by restoring the backed-up Compose file and config while stopped, +then start only the bot. Preserve the pre-deployment plugin data backup for recovery; +do not overwrite new chat history unless recovery specifically requires it. diff --git a/src/HuaJiBot.NET.Adapter.OneBot/CommandReader.cs b/src/HuaJiBot.NET.Adapter.OneBot/CommandReader.cs index 1ec38d9..6791deb 100644 --- a/src/HuaJiBot.NET.Adapter.OneBot/CommandReader.cs +++ b/src/HuaJiBot.NET.Adapter.OneBot/CommandReader.cs @@ -32,6 +32,9 @@ IEnumerable Parse() case ReplyMessageEntity { MessageId: var id }: yield return new ReaderReply(new(messageId: id)); break; + case ImageMessageEntity: + // Images are provided separately through GroupMessageEventArgs.ImageUrls. + break; default: service.LogDebug($"未转换消息类型：{element.GetType().Name}"); break; diff --git a/src/HuaJiBot.NET.Adapter.OneBot/OneBotMessageHandler.cs b/src/HuaJiBot.NET.Adapter.OneBot/OneBotMessageHandler.cs index 32772d4..eea7b15 100644 --- a/src/HuaJiBot.NET.Adapter.OneBot/OneBotMessageHandler.cs +++ b/src/HuaJiBot.NET.Adapter.OneBot/OneBotMessageHandler.cs @@ -211,6 +211,7 @@ public async Task ProcessMessageAsync(JObject json) _ => MemberType.Unknown, }, TextMessageLazy = new(() => ToPlainText(message)), + ImageUrls = ExtractImageUrls(message), } ); } @@ -233,6 +234,13 @@ public async Task ProcessMessageAsync(JObject json) #endif } + internal static string[] ExtractImageUrls(IEnumerable messages) => + messages.OfType() + .Select(image => string.IsNullOrWhiteSpace(image.Url) ? image.File : image.Url) + .Where(url => !string.IsNullOrWhiteSpace(url)) + .Distinct(StringComparer.Ordinal) + .ToArray(); + private static string ToPlainText(IEnumerable message) => string.Concat( message.Select(x => diff --git a/src/HuaJiBot.NET.Plugin.AIChat/Config/PluginConfig.cs b/src/HuaJiBot.NET.Plugin.AIChat/Config/PluginConfig.cs index 0297980..bdc82e6 100644 --- a/src/HuaJiBot.NET.Plugin.AIChat/Config/PluginConfig.cs +++ b/src/HuaJiBot.NET.Plugin.AIChat/Config/PluginConfig.cs @@ -6,6 +6,10 @@ public class PluginConfig : ConfigBase { public string SystemPrompt = "你是一个有用的AI助手"; public ModelConfig Model = new(); + /// Enable only for models that accept image content. + public bool SupportsVision = false; + public int MaxImagesPerRequest = 4; + public int MaxImageBytes = 8 * 1024 * 1024; /// /// Groups where the bot answers. Empty answers no group. diff --git a/src/HuaJiBot.NET.Plugin.AIChat/HuaJiBot.NET.Plugin.AIChat.csproj b/src/HuaJiBot.NET.Plugin.AIChat/HuaJiBot.NET.Plugin.AIChat.csproj index 79ab794..46a914a 100644 --- a/src/HuaJiBot.NET.Plugin.AIChat/HuaJiBot.NET.Plugin.AIChat.csproj +++ b/src/HuaJiBot.NET.Plugin.AIChat/HuaJiBot.NET.Plugin.AIChat.csproj @@ -11,6 +11,7 @@ + diff --git a/src/HuaJiBot.NET.Plugin.AIChat/PluginMain.cs b/src/HuaJiBot.NET.Plugin.AIChat/PluginMain.cs index c6aabfe..f05ee5a 100644 --- a/src/HuaJiBot.NET.Plugin.AIChat/PluginMain.cs +++ b/src/HuaJiBot.NET.Plugin.AIChat/PluginMain.cs @@ -4,7 +4,6 @@ using HuaJiBot.NET.Plugin.AIChat.Service; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -using Newtonsoft.Json; using System.Collections.Concurrent; namespace HuaJiBot.NET.Plugin.AIChat; @@ -18,7 +17,9 @@ private sealed record GroupSession(AgentSession Session, DateTimeOffset LastUsed private static readonly TimeSpan SessionIdleTimeout = TimeSpan.FromHours(1); private static readonly TimeSpan LlmTimeout = TimeSpan.FromSeconds(90); private readonly ConcurrentDictionary _sessions = new(); - private AgentConnector Connector => AgentConnector.Create(Service, Config.Model); + private AgentConnector Connector => AgentConnector.Create(Service, + Config.SupportsVision ? Config.Model with { Logging = false } : Config.Model); + private readonly VisionInput _visionInput = new(); private AIFunction[]? _tools; @@ -103,6 +104,10 @@ Events.GroupMessageEventArgs e Warn($"群组 {e.GroupId} 的 AI 请求超过 {LlmTimeout.TotalSeconds:0} 秒"); await e.Reply("模型响应超时，请稍后重试。"); } + catch (VisionInputException exception) + { + await e.Reply(exception.Message); + } finally { _busyGroups.TryRemove(e.GroupId, out _); @@ -116,21 +121,21 @@ private async Task InvokeLlmMessageCore( CancellationToken cancellationToken ) { - Info( - "调用AI消息\n\t" - + JsonConvert - .SerializeObject( - messages, - new JsonSerializerSettings - { - Formatting = Formatting.Indented, - NullValueHandling = NullValueHandling.Ignore, - } - ) - .Replace("\n", "\n\t") - ); + await _visionInput.PrepareAsync(messages, Config.SupportsVision, Config.MaxImagesPerRequest, + Config.MaxImageBytes, cancellationToken); + Info($"调用AI：{messages.Count} 条消息，{messages.Sum(message => message.Contents.OfType().Count())} 张图片"); var connector = Connector; - var session = await GetSessionAsync(connector, e.GroupId, cancellationToken); + AgentSession session; + if (messages.Any(message => message.Contents.OfType().Any())) + { + // Reply context is already reconstructed above. Do not retain image bytes across turns. + _sessions.TryRemove(e.GroupId, out _); + session = await connector.CreateSessionAsync(cancellationToken); + } + else + { + session = await GetSessionAsync(connector, e.GroupId, cancellationToken); + } var agent = connector.CreateAIAgentWithOptions( BuildSystemPrompt( systemPrompt, @@ -200,7 +205,8 @@ private async Task OnGroupMessageReceivedAsync(Events.GroupMessageEventArgs e) { if (!Service.AllRobots.Contains(atId)) return; //仅处理At机器人 - if (!reader.Input(out var restText, true) || string.IsNullOrWhiteSpace(restText)) + _ = reader.Input(out var restText, true); + if (string.IsNullOrWhiteSpace(restText) && e.ImageUrls.Length == 0) { try { @@ -221,6 +227,7 @@ private async Task OnGroupMessageReceivedAsync(Events.GroupMessageEventArgs e) new GroupMessage { Content = restText, + ImageUrls = e.ImageUrls, GroupId = e.GroupId, MessageId = e.MessageId, SenderId = e.SenderId, @@ -229,7 +236,7 @@ private async Task OnGroupMessageReceivedAsync(Events.GroupMessageEventArgs e) ReplyToMessageId = null, } ); - if (AsciiArtTools.TryRenderCommand(restText, out var asciiArt)) + if (e.ImageUrls.Length == 0 && AsciiArtTools.TryRenderCommand(restText!, out var asciiArt)) { // Send raw text. Markdown conversion changes FIGlet spacing and line breaks. foreach (var msgId in await e.Reply(asciiArt)) @@ -252,7 +259,7 @@ private async Task OnGroupMessageReceivedAsync(Events.GroupMessageEventArgs e) //调用LLM回复 await InvokeLlmMessage( Config.SystemPrompt, - [new ChatMessage(ChatRole.User, restText)], + [VisionInput.CreateMessage(restText, e.ImageUrls)], e ); } @@ -295,7 +302,7 @@ void PrependMessage(GroupMessage message) void FetchReply(string replyId) { var replyMessage = _history.GetMessage(replyId); - if (replyMessage is not null) + if (replyMessage is not null && replyMessage.GroupId == e.GroupId) { //获取到被回复的消息 PrependMessage(replyMessage); } @@ -335,16 +342,17 @@ void FetchReply(string replyId) prompts.Add( message.IsBot ? new ChatMessage(ChatRole.Assistant, message.Content) - : new ChatMessage(ChatRole.User, message.Content) + : VisionInput.CreateMessage(message.Content, message.ImageUrls) ); } - prompts.Add(new ChatMessage(ChatRole.User, text)); + prompts.Add(VisionInput.CreateMessage(text, e.ImageUrls)); //收到回复消息记录 _history.StoreMessage( new GroupMessage { Content = text, + ImageUrls = e.ImageUrls, GroupId = e.GroupId, MessageId = e.MessageId, SenderId = e.SenderId, diff --git a/src/HuaJiBot.NET.Plugin.AIChat/Service/VisionInput.cs b/src/HuaJiBot.NET.Plugin.AIChat/Service/VisionInput.cs new file mode 100644 index 0000000..5ff5957 --- /dev/null +++ b/src/HuaJiBot.NET.Plugin.AIChat/Service/VisionInput.cs @@ -0,0 +1,133 @@ +using System.Net; +using System.Net.Sockets; +using Microsoft.Extensions.AI; +using SixLabors.ImageSharp; + +namespace HuaJiBot.NET.Plugin.AIChat.Service; + +/// Downloads bounded image attachments without exposing URLs or image data in logs. +internal sealed class VisionInput +{ + private static readonly HttpClient SharedClient = new(new SocketsHttpHandler + { + AllowAutoRedirect = false, + ConnectCallback = async (context, cancellationToken) => + { + var addresses = await Dns.GetHostAddressesAsync(context.DnsEndPoint.Host, cancellationToken); + if (addresses.Length == 0 || addresses.Any(address => !IsPublicAddress(address))) + throw new VisionInputException("图片地址必须指向公共网络。"); + var socket = new Socket(SocketType.Stream, ProtocolType.Tcp); + try + { + await socket.ConnectAsync(addresses, context.DnsEndPoint.Port, cancellationToken); + return new NetworkStream(socket, ownsSocket: true); + } + catch + { + socket.Dispose(); + throw; + } + }, + UseProxy = false, + }) + { Timeout = TimeSpan.FromSeconds(20) }; + + private readonly HttpClient _client; + internal VisionInput(HttpClient? client = null) => _client = client ?? SharedClient; + + internal static ChatMessage CreateMessage(string? text, IEnumerable? images = null) + { + var urls = images?.Distinct(StringComparer.Ordinal).ToArray() ?? []; + var message = new ChatMessage(ChatRole.User, + urls.Length > 0 && string.IsNullOrWhiteSpace(text) ? "请描述这张图片中的内容。" : text ?? ""); + foreach (var url in urls) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || uri.Scheme is not ("https" or "http") + || !string.IsNullOrEmpty(uri.UserInfo)) + throw new VisionInputException("图片附件没有可用的 HTTP(S) 地址。"); + message.Contents.Add(new UriContent(uri, "image/*")); + } + return message; + } + + internal async Task PrepareAsync(IList messages, bool supportsVision, + int maxImages, int maxBytes, CancellationToken cancellationToken) + { + var imageCount = messages.Sum(message => message.Contents.OfType().Count()); + if (imageCount == 0) + return; + if (!supportsVision) + throw new VisionInputException("当前模型未启用图片理解，请用文字描述图片，或联系管理员启用视觉模型。"); + if (maxImages is < 1 or > 16 || maxBytes is < 1 or > 64 * 1024 * 1024) + throw new VisionInputException("图片限制配置无效，请联系管理员。"); + if (imageCount > maxImages) + throw new VisionInputException($"本次对话最多读取 {maxImages} 张图片，请减少附图或开始新的提问。"); + foreach (var message in messages) + { + var urls = message.Contents.OfType().ToArray(); + foreach (var content in urls) + message.Contents.Remove(content); + await AddImagesAsync(message, urls.Select(content => content.Uri.ToString()), maxBytes, cancellationToken); + } + } + + internal async Task AddImagesAsync( + ChatMessage message, IEnumerable urls, int maxBytes, CancellationToken cancellationToken) + { + foreach (var url in urls) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || uri.Scheme is not ("https" or "http") + || !string.IsNullOrEmpty(uri.UserInfo)) + throw new VisionInputException("图片附件没有可用的 HTTP(S) 地址。"); + try + { + using var response = await _client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + if (!response.IsSuccessStatusCode) + throw new VisionInputException("图片无法下载，请重新发送图片。"); + if (response.Content.Headers.ContentLength > maxBytes) + throw new VisionInputException("图片超过大小限制，请压缩后重试。"); + using var downloadTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + downloadTimeout.CancelAfter(TimeSpan.FromSeconds(20)); + await using var stream = await response.Content.ReadAsStreamAsync(downloadTimeout.Token); + using var data = new MemoryStream(); + var buffer = new byte[8192]; + int read; + while ((read = await stream.ReadAsync(buffer, downloadTimeout.Token)) != 0) + { + if (data.Length + read > maxBytes) + throw new VisionInputException("图片超过大小限制，请压缩后重试。"); + data.Write(buffer, 0, read); + } + var bytes = data.ToArray(); + var format = Image.DetectFormat(bytes); + if (format.DefaultMimeType is not ("image/png" or "image/jpeg" or "image/webp" or "image/gif")) + throw new VisionInputException("暂不支持该图片格式，请发送 PNG、JPEG、WebP 或 GIF。"); + message.Contents.Add(new DataContent(bytes, format.DefaultMimeType)); + } + catch (VisionInputException) { throw; } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (Exception exception) when (exception is HttpRequestException or OperationCanceledException or UnknownImageFormatException or InvalidImageContentException) + { + throw new VisionInputException("图片读取失败或超时，请重新发送图片。"); + } + } + } + + internal static bool IsPublicAddress(IPAddress address) + { + if (address.IsIPv4MappedToIPv6) + address = address.MapToIPv4(); + if (IPAddress.IsLoopback(address)) + return false; + var bytes = address.GetAddressBytes(); + if (address.AddressFamily == AddressFamily.InterNetworkV6) + return (bytes[0] & 0xe0) == 0x20; // Global unicast only. + return bytes[0] is not (0 or 10 or 127) && bytes[0] < 224 + && !(bytes[0] == 169 && bytes[1] == 254) + && !(bytes[0] == 172 && bytes[1] is >= 16 and <= 31) + && !(bytes[0] == 192 && bytes[1] == 168) + && !(bytes[0] == 100 && bytes[1] is >= 64 and <= 127); + } +} + +internal sealed class VisionInputException(string message) : Exception(message); diff --git a/src/HuaJiBot.NET.UnitTest/AIChatVisionIntegrationTest.cs b/src/HuaJiBot.NET.UnitTest/AIChatVisionIntegrationTest.cs new file mode 100644 index 0000000..e2eef65 --- /dev/null +++ b/src/HuaJiBot.NET.UnitTest/AIChatVisionIntegrationTest.cs @@ -0,0 +1,106 @@ +using System.Reflection; +using HuaJiBot.NET.Adapter.OneBot; +using HuaJiBot.NET.Bot; +using HuaJiBot.NET.Config; +using HuaJiBot.NET.Logger; +using HuaJiBot.NET.Plugin.AIChat; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace HuaJiBot.NET.UnitTest; + +/// Real provider test with fixture OneBot events; never posts messages to QQ. +internal class AIChatVisionIntegrationTest +{ + [Test] + [Explicit("Requires HUAJIBOT_VISION_TEST_CONFIG and calls its real AI provider")] + public async Task RealModel_OneBotFixture_ImageFollowupTextAndDisabledVision() + { + var path = Environment.GetEnvironmentVariable("HUAJIBOT_VISION_TEST_CONFIG"); + Assert.That(path, Is.Not.Null.And.Not.Empty); + var config = JsonConvert.DeserializeObject(File.ReadAllText(path!))!; + var directory = Path.Combine(Path.GetTempPath(), "huajibot-vision-test-" + Guid.NewGuid().ToString("N")); + var adapter = new FixtureAdapter(directory) { Logger = new ConsoleLogger() }; + await new Internal().SetupServiceAsync(adapter, config); + var plugin = new PluginMain(); + typeof(PluginBase).GetProperty(nameof(PluginBase.Service))!.SetValue(plugin, adapter); + typeof(PluginBase).GetProperty(nameof(PluginBase.Name))!.SetValue(plugin, "AIChat"); + JsonConvert.PopulateObject(config.Plugins["AIChat"].ToString(), plugin.Config); + plugin.Config.GroupIds = ["vision-fixture"]; + plugin.Config.McpServers = []; + plugin.Config.SupportsVision = true; + plugin.Config.SystemPrompt = "请根据图片回答，使用中文，简明扼要。"; + await (Task)typeof(PluginMain).GetMethod("InitializeAsync", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(plugin, null)!; + var handler = new OneBotMessageHandler(new OneBotApi(adapter, _ => { }), adapter); + const string image = "https://raw.githubusercontent.com/github/explore/main/topics/csharp/csharp.png"; + try + { + async Task SendAsync(int id, params JObject[] segments) + { + adapter.Next = new(TaskCreationOptions.RunContinuationsAsynchronously); + await handler.ProcessMessageAsync(new JObject + { + ["post_type"] = "message", + ["message_type"] = "group", + ["group_id"] = "vision-fixture", + ["user_id"] = "fixture-user", + ["message_id"] = id, + ["sender"] = new JObject { ["nickname"] = "vision fixture", ["role"] = "member" }, + ["message"] = new JArray(segments), + }); + return await adapter.Next.Task.WaitAsync(TimeSpan.FromSeconds(100)); + } + static JObject Segment(string type, JObject data) => new() { ["type"] = type, ["data"] = data }; + var mention = Segment("at", new JObject { ["qq"] = "fixture-bot" }); + var attachment = Segment("image", new JObject { ["url"] = image, ["file"] = "fixture.image" }); + + var first = await SendAsync(1, mention, attachment); + Console.WriteLine("IMAGE_ONLY: " + first); + Assert.That(first, Does.Contain("紫").Or.Contain("C#").Or.Contain("C Sharp")); + await Task.Delay(100); // Allow the event handler to persist the captured reply. + + var followup = await SendAsync(2, Segment("reply", new JObject { ["id"] = "fixture-reply-1" }), + Segment("text", new JObject { ["text"] = "之前图片的主色是什么？只回复颜色。" })); + Console.WriteLine("IMAGE_FOLLOWUP: " + followup); + Assert.That(followup, Does.Contain("紫")); + await Task.Delay(100); + + var text = await SendAsync(3, mention, + Segment("text", new JObject { ["text"] = "只回复 VISION_TEXT_OK" })); + Console.WriteLine("TEXT_ONLY: " + text); + Assert.That(text, Does.Contain("VISION_TEXT_OK")); + await Task.Delay(100); + + plugin.Config.SupportsVision = false; + var disabled = await SendAsync(4, mention, attachment); + Console.WriteLine("VISION_DISABLED: " + disabled); + Assert.That(disabled, Does.Contain("未启用")); + } + finally + { + typeof(PluginMain).GetMethod("Unload", BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(plugin, null); + Directory.Delete(directory, recursive: true); + } + } + + private sealed class FixtureAdapter(string directory) : OneBotAdapter("ws://unused", null) + { + internal TaskCompletionSource Next = new(); + private int _replies; + public override string[] AllRobots => ["fixture-bot"]; + protected override Task SetupServiceAsyncCore() => Task.CompletedTask; + protected override string GetPluginDataPathCore() => directory; + public override Task SendGroupMessageAsync(string? robotId, string targetGroup, params SendingMessageBase[] messages) + { + Next.TrySetResult(string.Concat(messages.OfType().Select(message => message.Text))); + return Task.FromResult(new[] { "fixture-reply-" + ++_replies }); + } + public override Task SendRichMessageAsync(string? robotId, string targetGroup, RichContent content, + Func> fallback) + { + Next.TrySetResult(content.Markdown); + return Task.FromResult(new[] { "fixture-reply-" + ++_replies }); + } + } +} diff --git a/src/HuaJiBot.NET.UnitTest/VisionInputTest.cs b/src/HuaJiBot.NET.UnitTest/VisionInputTest.cs new file mode 100644 index 0000000..bbd8816 --- /dev/null +++ b/src/HuaJiBot.NET.UnitTest/VisionInputTest.cs @@ -0,0 +1,159 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net; +using System.Text; +using HuaJiBot.NET.Adapter.OneBot; +using HuaJiBot.NET.Adapter.OneBot.Message; +using HuaJiBot.NET.Adapter.OneBot.Message.Entity; +using HuaJiBot.NET.DataBase; +using HuaJiBot.NET.Plugin.AIChat.Service; +using LiteDB; +using Microsoft.Extensions.AI; +using Newtonsoft.Json.Linq; +using OpenAI; + +namespace HuaJiBot.NET.UnitTest; + +internal class VisionInputTest +{ + private static readonly byte[] Png = Convert.FromBase64String( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jG1sAAAAASUVORK5CYII="); + + [Test] + public void OneBotAttachments_DoNotTreatTypedUrlsAsImages() + { + var messages = JArray.Parse(""" + [{"type":"at","data":{"qq":"123"}}, + {"type":"text","data":{"text":"https://example.com/typed.png"}}, + {"type":"image","data":{"file":"qq.image","url":"https://example.com/actual.png"}}, + {"type":"image","data":{"file":"https://example.com/fallback.png"}}] + """).ToObject>()!; + Assert.That(OneBotMessageHandler.ExtractImageUrls(messages), + Is.EqualTo(new[] { "https://example.com/actual.png", "https://example.com/fallback.png" })); + var reader = new OneBotCommandReader(new TestAdapter(), messages); + Assert.That(reader.At(out var target), Is.True); + Assert.That(target, Is.EqualTo("123")); + Assert.That(reader.Input(out var text, true), Is.True); + Assert.That(text, Is.EqualTo("https://example.com/typed.png")); + } + + [Test] + public async Task ImageOnly_DefaultPromptAndRealImageContent() + { + using var http = new HttpClient(new ResponseHandler(() => new HttpResponseMessage(HttpStatusCode.OK) + { Content = new ByteArrayContent(Png) })); + var message = VisionInput.CreateMessage(null, ["https://example.com/a.png"]); + await new VisionInput(http).PrepareAsync([message], true, 4, 1024, default); + Assert.That(message.Text, Does.Contain("图片")); + Assert.That(message.Contents.OfType().Single().MediaType, Is.EqualTo("image/png")); + Assert.That(message.Contents.OfType().Single().Data.ToArray(), Is.EqualTo(Png)); + } + + [Test] + public void VisionDisabled_RejectsBeforeDownloading() + { + var message = VisionInput.CreateMessage("看图", ["https://example.com/a.png"]); + var error = Assert.ThrowsAsync(() => + new VisionInput().PrepareAsync([message], false, 4, 1024, default)); + Assert.That(error!.Message, Does.Contain("未启用")); + } + + [Test] + public void ImageLimitIncludesHistory() + { + var history = VisionInput.CreateMessage("之前的图", ["https://example.com/a.png"]); + var current = VisionInput.CreateMessage("新的图", ["https://example.com/b.png"]); + Assert.ThrowsAsync(() => + new VisionInput().PrepareAsync([history, current], true, 1, 1024, default)); + } + + [TestCase("file:///etc/passwd")] + [TestCase("https://user:password@example.com/a.png")] + public void InvalidAttachmentSchemeOrCredentialsRejected(string url) => + Assert.Throws(() => VisionInput.CreateMessage("", [url])); + + [TestCase("127.0.0.1", false)] + [TestCase("10.0.0.1", false)] + [TestCase("169.254.169.254", false)] + [TestCase("100.66.88.34", false)] + [TestCase("::1", false)] + [TestCase("::ffff:192.168.1.1", false)] + [TestCase("fd00::1", false)] + [TestCase("8.8.8.8", true)] + public void DownloadsDoNotConnectToPrivateAddresses(string address, bool expected) => + Assert.That(VisionInput.IsPublicAddress(IPAddress.Parse(address)), Is.EqualTo(expected)); + + [TestCase(404)] + [TestCase(302)] + public void FailedOrRedirectedDownloadHasActionableError(int status) + { + using var http = new HttpClient(new ResponseHandler(() => new HttpResponseMessage((HttpStatusCode)status))); + Assert.ThrowsAsync(() => new VisionInput(http).PrepareAsync( + [VisionInput.CreateMessage("看图", ["https://example.com/a.png"])], true, 4, 1024, default)); + } + + [Test] + public void OversizedAndNonImageContentRejected() + { + foreach (var bytes in new[] { new byte[1025], Encoding.UTF8.GetBytes("not an image") }) + { + using var http = new HttpClient(new ResponseHandler(() => new HttpResponseMessage(HttpStatusCode.OK) + { Content = new ByteArrayContent(bytes) })); + Assert.ThrowsAsync(() => new VisionInput(http).PrepareAsync( + [VisionInput.CreateMessage("看图", ["https://example.com/a.png"])], true, 4, 1024, default)); + } + } + + [Test] + public void OldHistoryLoadsWithoutAttachmentsAndNewHistoryRetainsThem() + { + var mapper = new BsonMapper(); + var old = mapper.ToObject(new BsonDocument + { ["_id"] = "old", ["GroupId"] = "group", ["Content"] = "text" }); + Assert.That(old.ImageUrls, Is.Empty); + old.ImageUrls = ["https://example.com/a.png"]; + var restored = mapper.ToObject(mapper.ToDocument(old)); + Assert.That(restored.ImageUrls, Is.EqualTo(old.ImageUrls)); + } + + [Test] + public async Task OpenAICompatibleRequestContainsImageBlockInsteadOfPlainUrl() + { + var handler = new ChatHandler(); + using var http = new HttpClient(handler); + var client = new OpenAIClient(new ApiKeyCredential("test"), new OpenAIClientOptions + { + Endpoint = new Uri("https://example.com/v1"), + Transport = new HttpClientPipelineTransport(http), + }).GetChatClient("vision-test").AsIChatClient(); + var message = VisionInput.CreateMessage("左右是什么颜色？"); + message.Contents.Add(new DataContent(Png, "image/png")); + await client.GetResponseAsync([message]); + var content = JObject.Parse(handler.Body!)["messages"]![0]!["content"]!; + Assert.That(content[0]!["text"]!.Value(), Is.EqualTo("左右是什么颜色？")); + Assert.That(content[1]!["type"]!.Value(), Is.EqualTo("image_url")); + Assert.That(content[1]!["image_url"]!["url"]!.Value(), Does.StartWith("data:image/png;base64,")); + } + + private sealed class ResponseHandler(Func response) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(response()); + } + + private sealed class ChatHandler : HttpMessageHandler + { + internal string? Body; + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Body = await request.Content!.ReadAsStringAsync(cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(""" + {"id":"test","object":"chat.completion","created":1,"model":"vision-test", + "choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]} + """, Encoding.UTF8, "application/json"), + }; + } + } +} diff --git a/src/HuaJiBot.NET/DataBase/MessageHistory.cs b/src/HuaJiBot.NET/DataBase/MessageHistory.cs index 43e03f0..bae8794 100644 --- a/src/HuaJiBot.NET/DataBase/MessageHistory.cs +++ b/src/HuaJiBot.NET/DataBase/MessageHistory.cs @@ -12,6 +12,7 @@ public class GroupMessage public required string SenderName { get; set; } = ""; public DateTime Timestamp { get; set; } = DateTime.Now; public required string? Content { get; set; } = ""; + public string[] ImageUrls { get; set; } = []; public required bool IsBot { get; set; } = false; public required string? ReplyToMessageId { get; set; } = null; } diff --git a/src/HuaJiBot.NET/Events/GroupMessageEventArgs.cs b/src/HuaJiBot.NET/Events/GroupMessageEventArgs.cs index e036282..1dc5ec9 100644 --- a/src/HuaJiBot.NET/Events/GroupMessageEventArgs.cs +++ b/src/HuaJiBot.NET/Events/GroupMessageEventArgs.cs @@ -46,6 +46,8 @@ Func> getGroupName /// Sender role in the group; Unknown when the adapter does not provide it. public MemberType SenderMemberType { get; init; } + /// Image attachments, distinct from URLs typed as text. + public string[] ImageUrls { get; init; } = []; public required Lazy TextMessageLazy { get; init; } public string TextMessage => TextMessageLazy.Value; public CommandReader CommandReader => createCommandReader(); From 0d15e6fd50527fc9f55b09403939b1138b2df171 Mon Sep 17 00:00:00 2001 From: Yuna-Celisse <29951517@qq.com> Date: Fri, 2 Oct 2026 21:42:14 +0800 Subject: [PATCH 2/2] fix(deploy): preserve executable mode for cross-published CLI --- deploy/aichat-multimodal/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/aichat-multimodal/Dockerfile b/deploy/aichat-multimodal/Dockerfile index ba5eecc..7b78362 100644 --- a/deploy/aichat-multimodal/Dockerfile +++ b/deploy/aichat-multimodal/Dockerfile @@ -3,4 +3,5 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} COPY cli/ /app/bin/ +COPY --chmod=755 cli/HuaJiBot.NET.CLI /app/bin/HuaJiBot.NET.CLI COPY aichat/HuaJiBot.NET.Plugin.AIChat.dll /app/plugins/HuaJiBot.NET.Plugin.AIChat.dll