sevenhill/QMAX
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up legacy outgoing attachment duplicates

Browse Source
This commit is contained in:
sevenhill
2026-07-05 00:26:18 +03:00
parent b022218a1f
commit b518f22a09
14 changed files with 407 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -20,6 +20,8 @@ local.properties
# QMAX runtime data and secrets
data/
!server/QMax.Api/Data/
!server/QMax.Api/Data/**
storage/
secrets/
.env
server/QMax.Api/Data/Entities/Chat.cs
+19
View File
@@ -0,0 +1,19 @@
namespace QMax.Api.Data.Entities;
public sealed class Chat
{
public Guid Id { get; set; } = Guid.NewGuid();
public string? ExternalId { get; set; }
public ChatKind Kind { get; set; } = ChatKind.MaxDialog;
public string Title { get; set; } = "MAX chat";
public string? AvatarUrl { get; set; }
public string? LastMessagePreview { get; set; }
public DateTimeOffset? LastMessageAt { get; set; }
public int UnreadCount { get; set; }
public bool IsPinned { get; set; }
public bool IsMuted { get; set; }
public bool IsArchived { get; set; }
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow;
public ICollection<Message> Messages { get; set; } = new List<Message>();
}
server/QMax.Api/Data/Entities/Enums.cs
+36
View File
@@ -0,0 +1,36 @@
namespace QMax.Api.Data.Entities;
public enum ChatKind
{
Direct = 0,
Group = 1,
Channel = 2,
MaxDialog = 3
}
public enum MessageDirection
{
Incoming = 0,
Outgoing = 1,
System = 2
}
public enum MessageDeliveryState
{
Sending = 0,
Sent = 1,
Delivered = 2,
Read = 3,
Failed = 4
}
public enum AttachmentKind
{
File = 0,
VoiceNote = 1,
Image = 2,
Video = 3,
Sticker = 4,
Gif = 5,
Contact = 6
}
server/QMax.Api/Data/Entities/MaxAccountState.cs
+12
View File
@@ -0,0 +1,12 @@
namespace QMax.Api.Data.Entities;
public sealed class MaxAccountState
{
public int Id { get; set; } = 1;
public string PhoneNumber { get; set; } = "";
public string Status { get; set; } = "NotStarted";
public bool IsAuthorized { get; set; }
public string? LastUrl { get; set; }
public string? LastError { get; set; }
public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow;
}
server/QMax.Api/Data/Entities/Message.cs
+23
View File
@@ -0,0 +1,23 @@
namespace QMax.Api.Data.Entities;
public sealed class Message
{
public Guid Id { get; set; } = Guid.NewGuid();
public Guid ChatId { get; set; }
public Chat? Chat { get; set; }
public string? ExternalId { get; set; }
public string? SenderExternalId { get; set; }
public string? SenderName { get; set; }
public MessageDirection Direction { get; set; }
public string? Text { get; set; }
public DateTimeOffset SentAt { get; set; } = DateTimeOffset.UtcNow;
public DateTimeOffset? EditedAt { get; set; }
public DateTimeOffset? DeletedAt { get; set; }
public Guid? ReplyToMessageId { get; set; }
public string? ForwardedFrom { get; set; }
public MessageDeliveryState DeliveryState { get; set; } = MessageDeliveryState.Sent;
public string? Error { get; set; }
public string? MediaAlbumId { get; set; }
public ICollection<MessageAttachment> Attachments { get; set; } = new List<MessageAttachment>();
public ICollection<MessageReaction> Reactions { get; set; } = new List<MessageReaction>();
}
server/QMax.Api/Data/Entities/MessageAttachment.cs
+18
View File
@@ -0,0 +1,18 @@
namespace QMax.Api.Data.Entities;
public sealed class MessageAttachment
{
public Guid Id { get; set; } = Guid.NewGuid();
public Guid MessageId { get; set; }
public Message? Message { get; set; }
public string? ExternalId { get; set; }
public string OriginalFileName { get; set; } = "file";
public string StorageFileName { get; set; } = "";
public string ContentType { get; set; } = "application/octet-stream";
public long FileSizeBytes { get; set; }
public string? Sha256 { get; set; }
public AttachmentKind Kind { get; set; } = AttachmentKind.File;
public int SortOrder { get; set; }
public string? RemoteUrl { get; set; }
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
}
server/QMax.Api/Data/Entities/MessageReaction.cs
+12
View File
@@ -0,0 +1,12 @@
namespace QMax.Api.Data.Entities;
public sealed class MessageReaction
{
public Guid Id { get; set; } = Guid.NewGuid();
public Guid MessageId { get; set; }
public Message? Message { get; set; }
public string Emoji { get; set; } = "";
public string ActorKey { get; set; } = "self";
public string? ActorName { get; set; }
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
}
server/QMax.Api/Data/Entities/PushDevice.cs
+12
View File
@@ -0,0 +1,12 @@
namespace QMax.Api.Data.Entities;
public sealed class PushDevice
{
public Guid Id { get; set; } = Guid.NewGuid();
public Guid UserId { get; set; }
public User? User { get; set; }
public string FirebaseToken { get; set; } = "";
public string Platform { get; set; } = "android";
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow;
}
server/QMax.Api/Data/Entities/User.cs
+12
View File
@@ -0,0 +1,12 @@
namespace QMax.Api.Data.Entities;
public sealed class User
{
public Guid Id { get; set; } = Guid.NewGuid();
public string DisplayName { get; set; } = "QMAX Owner";
public string? PhoneNumber { get; set; }
public string? AvatarPath { get; set; }
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow;
public ICollection<UserSession> Sessions { get; set; } = new List<UserSession>();
}
server/QMax.Api/Data/Entities/UserSession.cs
+14
View File
@@ -0,0 +1,14 @@
namespace QMax.Api.Data.Entities;
public sealed class UserSession
{
public Guid Id { get; set; } = Guid.NewGuid();
public Guid UserId { get; set; }
public User? User { get; set; }
public string DeviceName { get; set; } = "Android";
public string RefreshTokenHash { get; set; } = "";
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
public DateTimeOffset LastSeenAt { get; set; } = DateTimeOffset.UtcNow;
public DateTimeOffset ExpiresAt { get; set; } = DateTimeOffset.UtcNow.AddDays(90);
public DateTimeOffset? RevokedAt { get; set; }
}
server/QMax.Api/Data/QMaxDatabaseCleanup.cs
+93
View File
@@ -0,0 +1,93 @@
using Microsoft.EntityFrameworkCore;
using QMax.Api.Data.Entities;
namespace QMax.Api.Data;
public static class QMaxDatabaseCleanup
{
public static async Task RemoveLegacyWebFileAttachmentEchoesAsync(QMaxDbContext db)
{
var connection = db.Database.GetDbConnection();
if (connection.State != System.Data.ConnectionState.Open)
{
await connection.OpenAsync();
}
await db.Database.ExecuteSqlRawAsync("DROP TABLE IF EXISTS temp.LegacyWebFileMessageIds;");
await db.Database.ExecuteSqlRawAsync("""
CREATE TEMP TABLE LegacyWebFileMessageIds (
Id TEXT NOT NULL PRIMARY KEY
);
""");
await db.Database.ExecuteSqlRawAsync("""
INSERT OR IGNORE INTO LegacyWebFileMessageIds (Id)
SELECT DISTINCT legacy.Id
FROM Messages AS legacy
INNER JOIN MessageAttachments AS legacyAttachment ON legacyAttachment.MessageId = legacy.Id
WHERE legacy.Direction = 'Outgoing'
AND legacy.ExternalId LIKE 'web-file-%'
AND legacyAttachment.Sha256 IS NOT NULL
AND legacyAttachment.Sha256 <> ''
AND EXISTS (
SELECT 1
FROM Messages AS confirmed
INNER JOIN MessageAttachments AS confirmedAttachment ON confirmedAttachment.MessageId = confirmed.Id
WHERE confirmed.ChatId = legacy.ChatId
AND confirmed.Id <> legacy.Id
AND confirmed.Direction = 'Outgoing'
AND confirmed.ExternalId IS NOT NULL
AND confirmed.ExternalId NOT LIKE 'web-file-%'
AND confirmedAttachment.Sha256 = legacyAttachment.Sha256
);
""");
await db.Database.ExecuteSqlRawAsync("""
DELETE FROM MessageAttachments
WHERE MessageId IN (SELECT Id FROM LegacyWebFileMessageIds);
""");
await db.Database.ExecuteSqlRawAsync("""
DELETE FROM Messages
WHERE Id IN (SELECT Id FROM LegacyWebFileMessageIds);
""");
await db.Database.ExecuteSqlRawAsync("DROP TABLE IF EXISTS temp.LegacyWebFileMessageIds;");
}
public static async Task ClearUnreadCountsForLatestOutgoingChatsAsync(QMaxDbContext db)
{
var unreadChats = await db.Chats
.Where(chat => chat.UnreadCount > 0)
.ToListAsync();
if (unreadChats.Count == 0)
{
return;
}
var changed = false;
foreach (var chat in unreadChats)
{
var chatMessages = await db.Messages
.Where(message => message.ChatId == chat.Id && message.DeletedAt == null)
.Select(message => new { message.Direction, message.SentAt, message.Id })
.ToListAsync();
var latestMessage = chatMessages
.OrderByDescending(message => message.SentAt)
.ThenByDescending(message => message.Id)
.FirstOrDefault();
if (latestMessage?.Direction != MessageDirection.Outgoing)
{
continue;
}
chat.UnreadCount = 0;
chat.UpdatedAt = DateTimeOffset.UtcNow;
changed = true;
}
if (changed)
{
await db.SaveChangesAsync();
}
}
}
server/QMax.Api/Data/QMaxDbContext.cs
+57
View File
@@ -0,0 +1,57 @@
using Microsoft.EntityFrameworkCore;
using QMax.Api.Data.Entities;
namespace QMax.Api.Data;
public sealed class QMaxDbContext(DbContextOptions<QMaxDbContext> options) : DbContext(options)
{
public DbSet<User> Users => Set<User>();
public DbSet<UserSession> UserSessions => Set<UserSession>();
public DbSet<Chat> Chats => Set<Chat>();
public DbSet<Message> Messages => Set<Message>();
public DbSet<MessageAttachment> MessageAttachments => Set<MessageAttachment>();
public DbSet<MessageReaction> MessageReactions => Set<MessageReaction>();
public DbSet<PushDevice> PushDevices => Set<PushDevice>();
public DbSet<MaxAccountState> MaxAccountStates => Set<MaxAccountState>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<User>().HasIndex(x => x.PhoneNumber);
modelBuilder.Entity<UserSession>().HasIndex(x => x.RefreshTokenHash).IsUnique();
modelBuilder.Entity<Chat>().HasIndex(x => x.ExternalId).IsUnique();
modelBuilder.Entity<Message>().HasIndex(x => new { x.ChatId, x.SentAt });
modelBuilder.Entity<Message>().HasIndex(x => x.ExternalId);
modelBuilder.Entity<MessageAttachment>().HasIndex(x => x.StorageFileName).IsUnique();
modelBuilder.Entity<MessageAttachment>().HasIndex(x => new { x.MessageId, x.ExternalId });
modelBuilder.Entity<MessageReaction>().HasIndex(x => new { x.MessageId, x.ActorKey }).IsUnique();
modelBuilder.Entity<PushDevice>().HasIndex(x => x.FirebaseToken).IsUnique();
modelBuilder.Entity<Chat>()
.Property(x => x.Kind)
.HasConversion<string>();
modelBuilder.Entity<Message>()
.Property(x => x.Direction)
.HasConversion<string>();
modelBuilder.Entity<Message>()
.Property(x => x.DeliveryState)
.HasConversion<string>();
modelBuilder.Entity<MessageAttachment>()
.Property(x => x.Kind)
.HasConversion<string>();
modelBuilder.Entity<Message>()
.HasMany(x => x.Attachments)
.WithOne(x => x.Message)
.HasForeignKey(x => x.MessageId)
.OnDelete(DeleteBehavior.Cascade);
modelBuilder.Entity<Message>()
.HasMany(x => x.Reactions)
.WithOne(x => x.Message)
.HasForeignKey(x => x.MessageId)
.OnDelete(DeleteBehavior.Cascade);
}
}
server/QMax.Api/Program.cs
+2
View File
@@ -161,6 +161,8 @@ static async Task EnsureCompatibilitySchemaAsync(QMaxDbContext db)
}
await RemoveDuplicateMessageAttachmentsAsync(db);
await QMaxDatabaseCleanup.RemoveLegacyWebFileAttachmentEchoesAsync(db);
await QMaxDatabaseCleanup.ClearUnreadCountsForLatestOutgoingChatsAsync(db);
await db.Database.ExecuteSqlRawAsync("DROP INDEX IF EXISTS IX_MessageAttachments_MessageId_ExternalId;");
await db.Database.ExecuteSqlRawAsync("""
CREATE UNIQUE INDEX IF NOT EXISTS IX_MessageAttachments_MessageId_ExternalId
tests/QMax.Tests/ApiSmokeTests.cs
+95 -2
View File
@@ -712,7 +712,7 @@ public sealed class ApiSmokeTests : IDisposable
{
Message = message,
OriginalFileName = "sent-photo.jpg",
StorageFileName = "sent-photo.jpg",
StorageFileName = $"{Guid.NewGuid():N}-sent-photo.jpg",
ContentType = "image/jpeg",
FileSizeBytes = 1024,
Kind = AttachmentKind.Image
@@ -791,6 +791,94 @@ public sealed class ApiSmokeTests : IDisposable
}
}
[Fact]
public async Task DatabaseCleanupRemovesLegacyWebFileAttachmentDuplicateAndClearsSelfUnread()
{
var databasePath = Path.Combine(Path.GetTempPath(), $"qmax-legacy-web-file-test-{Guid.NewGuid():N}.db");
var dbOptions = new DbContextOptionsBuilder<QMaxDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options;
var chatId = Guid.NewGuid();
const string attachmentHash = "same-photo-sha";
var legacySentAt = DateTimeOffset.UtcNow.AddMinutes(-40);
try
{
await using (var db = new QMaxDbContext(dbOptions))
{
await db.Database.EnsureCreatedAsync();
db.Chats.Add(new Chat
{
Id = chatId,
Title = "Legacy Photo",
LastMessagePreview = "\u041c\u0435\u0434\u0438\u0430",
LastMessageAt = legacySentAt.AddMinutes(30),
UnreadCount = 2
});
var legacyMessage = new Message
{
ChatId = chatId,
ExternalId = "web-file-123",
Direction = MessageDirection.Outgoing,
SentAt = legacySentAt,
DeliveryState = MessageDeliveryState.Sent
};
legacyMessage.Attachments.Add(new MessageAttachment
{
OriginalFileName = "legacy.jpg",
StorageFileName = $"{Guid.NewGuid():N}.jpg",
ContentType = "image/jpeg",
FileSizeBytes = 10,
Sha256 = attachmentHash,
Kind = AttachmentKind.Image
});
var confirmedMessage = new Message
{
ChatId = chatId,
ExternalId = "domhist:real-photo",
Direction = MessageDirection.Outgoing,
SentAt = legacySentAt.AddMinutes(30),
DeliveryState = MessageDeliveryState.Sent
};
confirmedMessage.Attachments.Add(new MessageAttachment
{
OriginalFileName = "confirmed.jpg",
StorageFileName = $"{Guid.NewGuid():N}.jpg",
ContentType = "image/jpeg",
FileSizeBytes = 10,
Sha256 = attachmentHash,
Kind = AttachmentKind.Image
});
db.Messages.AddRange(legacyMessage, confirmedMessage);
await db.SaveChangesAsync();
}
await using (var cleanupDb = new QMaxDbContext(dbOptions))
{
await QMaxDatabaseCleanup.RemoveLegacyWebFileAttachmentEchoesAsync(cleanupDb);
await QMaxDatabaseCleanup.ClearUnreadCountsForLatestOutgoingChatsAsync(cleanupDb);
}
await using var verifyDb = new QMaxDbContext(dbOptions);
var messages = await verifyDb.Messages
.Where(message => message.ChatId == chatId)
.ToListAsync();
Assert.DoesNotContain(messages, message => message.ExternalId == "web-file-123");
Assert.Contains(messages, message => message.ExternalId == "domhist:real-photo");
var chat = await verifyDb.Chats.FindAsync(chatId);
Assert.NotNull(chat);
Assert.Equal(0, chat!.UnreadCount);
}
finally
{
DeleteSqliteDatabase(databasePath);
}
}
[Fact]
public async Task AdminStatusRequiresPairingCode()
{
@@ -832,9 +920,14 @@ public sealed class ApiSmokeTests : IDisposable
public void Dispose()
{
_factory.Dispose();
DeleteSqliteDatabase(_databasePath);
}
private static void DeleteSqliteDatabase(string databasePath)
{
foreach (var suffix in new[] { "", "-shm", "-wal" })
{
var path = _databasePath + suffix;
var path = databasePath + suffix;
if (File.Exists(path))
{
File.Delete(path);