From 5d33320e3d14089edddad5b08dec116fdc0df14a Mon Sep 17 00:00:00 2001 From: sevenhill Date: Sun, 5 Jul 2026 08:58:38 +0300 Subject: [PATCH] Merge outgoing photo echo attachments --- server/QMax.Api/Data/QMaxDatabaseCleanup.cs | 61 +++++ server/QMax.Api/Program.cs | 1 + .../QMax.Api/Services/MaxBridgeSyncService.cs | 67 ++++- tests/QMax.Tests/ApiSmokeTests.cs | 229 ++++++++++++++++++ 4 files changed, 355 insertions(+), 3 deletions(-) diff --git a/server/QMax.Api/Data/QMaxDatabaseCleanup.cs b/server/QMax.Api/Data/QMaxDatabaseCleanup.cs index 58426c3..6b18a3a 100644 --- a/server/QMax.Api/Data/QMaxDatabaseCleanup.cs +++ b/server/QMax.Api/Data/QMaxDatabaseCleanup.cs @@ -51,6 +51,66 @@ public static class QMaxDatabaseCleanup await db.Database.ExecuteSqlRawAsync("DROP TABLE IF EXISTS temp.LegacyWebFileMessageIds;"); } + public static async Task MergeOutgoingRemoteAttachmentEchoesAsync(QMaxDbContext db) + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + { + await connection.OpenAsync(); + } + + await db.Database.ExecuteSqlRawAsync("DROP TABLE IF EXISTS temp.OutgoingAttachmentEchoMerges;"); + await db.Database.ExecuteSqlRawAsync(""" + CREATE TEMP TABLE OutgoingAttachmentEchoMerges ( + LocalId TEXT NOT NULL PRIMARY KEY, + RemoteId TEXT NOT NULL, + ExternalId TEXT NULL, + RemoteUrl TEXT NULL + ); + """); + await db.Database.ExecuteSqlRawAsync(""" + INSERT OR IGNORE INTO OutgoingAttachmentEchoMerges (LocalId, RemoteId, ExternalId, RemoteUrl) + SELECT localAttachment.Id, + remoteAttachment.Id, + remoteAttachment.ExternalId, + remoteAttachment.RemoteUrl + FROM Messages AS message + INNER JOIN MessageAttachments AS localAttachment ON localAttachment.MessageId = message.Id + INNER JOIN MessageAttachments AS remoteAttachment ON remoteAttachment.MessageId = message.Id + WHERE message.Direction = 'Outgoing' + AND localAttachment.Id <> remoteAttachment.Id + AND localAttachment.SortOrder = remoteAttachment.SortOrder + AND localAttachment.Kind = remoteAttachment.Kind + AND (localAttachment.ExternalId IS NULL OR trim(localAttachment.ExternalId) = '') + AND (localAttachment.RemoteUrl IS NULL OR trim(localAttachment.RemoteUrl) = '') + AND localAttachment.StorageFileName NOT LIKE 'remote-%' + AND ( + (remoteAttachment.ExternalId IS NOT NULL AND trim(remoteAttachment.ExternalId) <> '') OR + (remoteAttachment.RemoteUrl IS NOT NULL AND trim(remoteAttachment.RemoteUrl) <> '') OR + remoteAttachment.StorageFileName LIKE 'remote-%' + ); + """); + await db.Database.ExecuteSqlRawAsync(""" + DELETE FROM MessageAttachments + WHERE Id IN (SELECT RemoteId FROM OutgoingAttachmentEchoMerges); + """); + await db.Database.ExecuteSqlRawAsync(""" + UPDATE MessageAttachments + SET ExternalId = COALESCE(NULLIF(ExternalId, ''), ( + SELECT ExternalId + FROM OutgoingAttachmentEchoMerges + WHERE LocalId = MessageAttachments.Id + )), + RemoteUrl = COALESCE(NULLIF(RemoteUrl, ''), ( + SELECT RemoteUrl + FROM OutgoingAttachmentEchoMerges + WHERE LocalId = MessageAttachments.Id + )) + WHERE Id IN (SELECT LocalId FROM OutgoingAttachmentEchoMerges); + """); + await db.Database.ExecuteSqlRawAsync("DROP TABLE IF EXISTS temp.OutgoingAttachmentEchoMerges;"); + } + public static async Task ClearUnreadCountsForLatestOutgoingChatsAsync(QMaxDbContext db) { var unreadChats = await db.Chats @@ -90,4 +150,5 @@ public static class QMaxDatabaseCleanup await db.SaveChangesAsync(); } } + } diff --git a/server/QMax.Api/Program.cs b/server/QMax.Api/Program.cs index d6f948c..276548a 100644 --- a/server/QMax.Api/Program.cs +++ b/server/QMax.Api/Program.cs @@ -162,6 +162,7 @@ static async Task EnsureCompatibilitySchemaAsync(QMaxDbContext db) await RemoveDuplicateMessageAttachmentsAsync(db); await QMaxDatabaseCleanup.RemoveLegacyWebFileAttachmentEchoesAsync(db); + await QMaxDatabaseCleanup.MergeOutgoingRemoteAttachmentEchoesAsync(db); await QMaxDatabaseCleanup.ClearUnreadCountsForLatestOutgoingChatsAsync(db); await db.Database.ExecuteSqlRawAsync("DROP INDEX IF EXISTS IX_MessageAttachments_MessageId_ExternalId;"); await db.Database.ExecuteSqlRawAsync(""" diff --git a/server/QMax.Api/Services/MaxBridgeSyncService.cs b/server/QMax.Api/Services/MaxBridgeSyncService.cs index 8bdf41c..fbd76d4 100644 --- a/server/QMax.Api/Services/MaxBridgeSyncService.cs +++ b/server/QMax.Api/Services/MaxBridgeSyncService.cs @@ -133,7 +133,7 @@ public sealed class MaxBridgeSyncService( update.LastMessageIsOutgoing != true && update.Messages.Count == 0 && HasListClockTime(update.LastMessageTimeText); - var lastKnownOutgoingMessages = !shouldLoadLastKnownMessage + var lastKnownOutgoingCandidates = !shouldLoadLastKnownMessage ? [] : await db.Messages .AsNoTracking() @@ -142,9 +142,11 @@ public sealed class MaxBridgeSyncService( x.ChatId == chat.Id && x.DeletedAt == null && x.Direction == MessageDirection.Outgoing) - .OrderByDescending(x => x.SentAt) - .Take(5) .ToListAsync(cancellationToken); + var lastKnownOutgoingMessages = lastKnownOutgoingCandidates + .OrderByDescending(x => x.SentAt) + .Take(5) + .ToList(); var lastKnownMessage = lastKnownOutgoingMessages .Select(x => new LastKnownMessageSnapshot( x.Direction, @@ -605,6 +607,17 @@ public sealed class MaxBridgeSyncService( foreach (var incomingAttachment in incomingAttachments) { + if (TryMergeOutgoingRemoteEchoIntoLocalAttachment( + message, + incoming, + knownAttachments, + incomingAttachment, + out var echoChanged)) + { + changed |= echoChanged; + continue; + } + if (HasAttachment(knownAttachments, incomingAttachment)) { continue; @@ -621,6 +634,54 @@ public sealed class MaxBridgeSyncService( return new MessageMergeResult(changed, addedAttachments); } + private static bool TryMergeOutgoingRemoteEchoIntoLocalAttachment( + Message message, + MaxMessageUpdate incoming, + IEnumerable attachments, + MaxAttachmentUpdate incomingAttachment, + out bool changed) + { + changed = false; + if (message.Direction != MessageDirection.Outgoing || !incoming.IsOutgoing) + { + return false; + } + + if (string.IsNullOrWhiteSpace(incomingAttachment.ExternalId) && + string.IsNullOrWhiteSpace(incomingAttachment.RemoteUrl)) + { + return false; + } + + var kind = ResolveKind(incomingAttachment) ?? AttachmentKind.File; + var localUpload = attachments.FirstOrDefault(attachment => + attachment.SortOrder == incomingAttachment.SortOrder && + attachment.Kind == kind && + string.IsNullOrWhiteSpace(attachment.ExternalId) && + string.IsNullOrWhiteSpace(attachment.RemoteUrl)); + + if (localUpload is null) + { + return false; + } + + if (!string.IsNullOrWhiteSpace(incomingAttachment.ExternalId) && + !string.Equals(localUpload.ExternalId, incomingAttachment.ExternalId, StringComparison.Ordinal)) + { + localUpload.ExternalId = incomingAttachment.ExternalId; + changed = true; + } + + if (!string.IsNullOrWhiteSpace(incomingAttachment.RemoteUrl) && + !string.Equals(localUpload.RemoteUrl, incomingAttachment.RemoteUrl, StringComparison.Ordinal)) + { + localUpload.RemoteUrl = incomingAttachment.RemoteUrl; + changed = true; + } + + return true; + } + private static IReadOnlyList DistinctAttachments(IReadOnlyList attachments) { if (attachments.Count <= 1) diff --git a/tests/QMax.Tests/ApiSmokeTests.cs b/tests/QMax.Tests/ApiSmokeTests.cs index 565fa58..6519167 100644 --- a/tests/QMax.Tests/ApiSmokeTests.cs +++ b/tests/QMax.Tests/ApiSmokeTests.cs @@ -293,6 +293,55 @@ public sealed class ApiSmokeTests : IDisposable Assert.Equal(bytes, await download.Content.ReadAsByteArrayAsync()); } + [Fact] + public async Task SyncMergesOutgoingAttachmentEchoIntoLocalUpload() + { + var bridge = new OutgoingAttachmentEchoMaxBridgeClient(); + using var factory = _factory.WithWebHostBuilder(builder => + { + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + services.RemoveAll(); + services.AddSingleton(bridge); + }); + }); + using var client = factory.CreateClient(); + await LoginAsync(client); + + var chat = await CreateDirectChatAsync(client, bridge.ChatExternalId, "Outgoing Echo"); + var localBytes = new byte[] { 0x89, 0x50, 0x4E, 0x47, 0x4C, 0x4F, 0x43, 0x41, 0x4C }; + using var form = new MultipartFormDataContent(); + using var fileContent = new ByteArrayContent(localBytes); + fileContent.Headers.ContentType = new MediaTypeHeaderValue("image/png"); + form.Add(fileContent, "file", "local-photo.png"); + + var upload = await client.PostAsync($"/api/chats/{chat.Id}/attachments", form); + await AssertStatusAsync(HttpStatusCode.OK, upload); + var sent = await upload.Content.ReadFromJsonAsync(JsonOptions); + Assert.NotNull(sent); + Assert.Equal(bridge.MessageExternalId, sent!.ExternalId); + Assert.Single(sent.Attachments); + + var sync = await client.PostAsync("/api/max/sync", null); + await AssertStatusAsync(HttpStatusCode.OK, sync); + + var messages = await client.GetFromJsonAsync($"/api/chats/{chat.Id}/messages", JsonOptions); + var echoed = Assert.Single(messages!, message => message.ExternalId == bridge.MessageExternalId); + var attachment = Assert.Single(echoed.Attachments); + Assert.Equal("local-photo.png", attachment.FileName); + + var download = await client.GetAsync(attachment.DownloadPath); + await AssertStatusAsync(HttpStatusCode.OK, download); + Assert.Equal(localBytes, await download.Content.ReadAsByteArrayAsync()); + + using var scope = factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var storedAttachment = await db.MessageAttachments.SingleAsync(x => x.MessageId == echoed.Id); + Assert.Equal(bridge.RemoteAttachmentExternalId, storedAttachment.ExternalId); + Assert.Equal(bridge.RemoteUrl, storedAttachment.RemoteUrl); + } + [Fact] public async Task ForwardMessageFlowCopiesTextAndAttachments() { @@ -879,6 +928,76 @@ public sealed class ApiSmokeTests : IDisposable } } + [Fact] + public async Task DatabaseCleanupMergesOutgoingRemoteAttachmentEchoes() + { + var databasePath = Path.Combine(Path.GetTempPath(), $"qmax-attachment-echo-test-{Guid.NewGuid():N}.db"); + var dbOptions = new DbContextOptionsBuilder() + .UseSqlite($"Data Source={databasePath};Pooling=False") + .Options; + var messageId = Guid.NewGuid(); + var localAttachmentId = Guid.NewGuid(); + + try + { + await using (var db = new QMaxDbContext(dbOptions)) + { + await db.Database.EnsureCreatedAsync(); + var chat = new Chat { Title = "Attachment Echo" }; + var message = new Message + { + Id = messageId, + Chat = chat, + ExternalId = "domhist:photo", + Direction = MessageDirection.Outgoing, + DeliveryState = MessageDeliveryState.Sent, + SentAt = DateTimeOffset.UtcNow + }; + message.Attachments.Add(new MessageAttachment + { + Id = localAttachmentId, + OriginalFileName = "local-photo.png", + StorageFileName = $"{Guid.NewGuid():N}.png", + ContentType = "image/png", + FileSizeBytes = 10, + Sha256 = "local-sha", + Kind = AttachmentKind.Image, + SortOrder = 0 + }); + message.Attachments.Add(new MessageAttachment + { + OriginalFileName = "max-image-1.jpg", + StorageFileName = $"{Guid.NewGuid():N}.jpg", + ExternalId = "dommedia:remote", + RemoteUrl = "https://max.test/media/photo.jpg", + ContentType = "image/jpeg", + FileSizeBytes = 8, + Sha256 = "remote-sha", + Kind = AttachmentKind.Image, + SortOrder = 0 + }); + db.Messages.Add(message); + await db.SaveChangesAsync(); + } + + await using (var cleanupDb = new QMaxDbContext(dbOptions)) + { + await QMaxDatabaseCleanup.MergeOutgoingRemoteAttachmentEchoesAsync(cleanupDb); + } + + await using var verifyDb = new QMaxDbContext(dbOptions); + var attachment = await verifyDb.MessageAttachments.SingleAsync(x => x.MessageId == messageId); + Assert.Equal(localAttachmentId, attachment.Id); + Assert.Equal("local-photo.png", attachment.OriginalFileName); + Assert.Equal("dommedia:remote", attachment.ExternalId); + Assert.Equal("https://max.test/media/photo.jpg", attachment.RemoteUrl); + } + finally + { + DeleteSqliteDatabase(databasePath); + } + } + [Fact] public async Task AdminStatusRequiresPairingCode() { @@ -1344,6 +1463,116 @@ public sealed class ApiSmokeTests : IDisposable } } + private sealed class OutgoingAttachmentEchoMaxBridgeClient : IMaxBridgeClient + { + public readonly string ChatExternalId = $"mock-outgoing-attachment-echo-{Guid.NewGuid():N}"; + public readonly string MessageExternalId = $"outgoing-photo-{Guid.NewGuid():N}"; + public readonly string RemoteAttachmentExternalId = $"dommedia:{Guid.NewGuid():N}"; + public readonly string RemoteUrl = $"https://max.test/media/{Guid.NewGuid():N}.jpg"; + private static readonly byte[] RemoteBytes = [0xFF, 0xD8, 0x52, 0x45, 0x4D, 0x4F, 0x54, 0x45, 0xFF, 0xD9]; + + public Task GetStatusAsync(CancellationToken cancellationToken) + { + return Task.FromResult(new MaxBridgeStatus("Test", true, "Authorized", "Ready", null, "Test", null, DateTimeOffset.UtcNow)); + } + + public Task BeginPhoneLoginAsync(string phoneNumber, CancellationToken cancellationToken) + { + return GetStatusAsync(cancellationToken); + } + + public Task SubmitLoginCodeAsync(string code, CancellationToken cancellationToken) + { + return GetStatusAsync(cancellationToken); + } + + public Task GetSnapshotAsync(CancellationToken cancellationToken) + { + return Task.FromResult(new MaxBrowserSnapshot(null, "Test", "", "", DateTimeOffset.UtcNow)); + } + + public Task> FetchUpdatesAsync(CancellationToken cancellationToken) + { + var sentAt = DateTimeOffset.UtcNow.AddSeconds(-2); + return Task.FromResult>( + [ + new MaxChatUpdate( + ChatExternalId, + "Outgoing Echo", + null, + sentAt, + [ + new MaxMessageUpdate( + MessageExternalId, + "self", + "You", + true, + "", + sentAt, + [ + new MaxAttachmentUpdate( + RemoteAttachmentExternalId, + "max-image-1.jpg", + "image/jpeg", + RemoteBytes.Length, + RemoteUrl, + nameof(AttachmentKind.Image), + 0) + ]) + ], + "\u041c\u0435\u0434\u0438\u0430", + true, + sentAt) + ]); + } + + public Task FetchChatHistoryAsync(string externalChatId, CancellationToken cancellationToken) + { + return Task.FromResult(null); + } + + public Task FetchChatPresenceAsync(string externalChatId, CancellationToken cancellationToken) + { + return Task.FromResult(null); + } + + public Task SendTextAsync(string externalChatId, string text, CancellationToken cancellationToken) + { + return Task.FromResult(new MaxSendResult(true, $"external-{Guid.NewGuid():N}", null)); + } + + public Task SendAttachmentAsync(string externalChatId, string path, string caption, CancellationToken cancellationToken) + { + return Task.FromResult(new MaxSendResult(true, MessageExternalId, null)); + } + + public Task EditMessageAsync(string externalChatId, string externalMessageId, string? currentText, string text, CancellationToken cancellationToken) + { + return Task.FromResult(new MaxActionResult(true, null)); + } + + public Task DeleteMessageAsync(string externalChatId, string externalMessageId, string? currentText, CancellationToken cancellationToken) + { + return Task.FromResult(new MaxActionResult(true, null)); + } + + public Task SetReactionAsync(string externalChatId, string externalMessageId, string? currentText, string emoji, CancellationToken cancellationToken) + { + return Task.FromResult(new MaxActionResult(true, null)); + } + + public Task ClearReactionAsync(string externalChatId, string externalMessageId, string? currentText, string emoji, CancellationToken cancellationToken) + { + return Task.FromResult(new MaxActionResult(true, null)); + } + + public Task DownloadMediaAsync(string remoteUrl, CancellationToken cancellationToken) + { + return Task.FromResult( + new MaxMediaDownload(new MemoryStream(RemoteBytes), "image/jpeg", RemoteBytes.Length)); + } + } + private sealed class DeferredRemoteMediaMaxBridgeClient : IMaxBridgeClient { public const string ChatExternalId = "mock-deferred-remote-media-chat";