From e50b99bcd2dce8c75521b54f6d77745212b5d897 Mon Sep 17 00:00:00 2001 From: sevenhill Date: Mon, 6 Jul 2026 13:37:07 +0300 Subject: [PATCH] Fix sticker capture and rendering --- .../main/java/xyz/kusoft/qmax/MainActivity.kt | 57 +++++- .../qmax/core/local/AttachmentDiskCache.kt | 1 + worker/src/server.js | 189 +++++++++++++++--- 3 files changed, 223 insertions(+), 24 deletions(-) diff --git a/android/app/src/main/java/xyz/kusoft/qmax/MainActivity.kt b/android/app/src/main/java/xyz/kusoft/qmax/MainActivity.kt index eb612a5..caa6036 100644 --- a/android/app/src/main/java/xyz/kusoft/qmax/MainActivity.kt +++ b/android/app/src/main/java/xyz/kusoft/qmax/MainActivity.kt @@ -2506,6 +2506,7 @@ private fun replyPreviewBody(message: MessageDto): String { val attachment = message.attachments.firstOrNull() ?: return "Message" return when { attachment.isEmojiAttachment() -> "Emoji" + attachment.isStickerAttachment() -> "Sticker" attachment.isImageAttachment() -> "Photo" attachment.isVideoAttachment() -> "Video" attachment.isVoiceAttachment() -> "Voice message" @@ -2689,6 +2690,13 @@ private fun AttachmentView( cachedImagePath, vm ) + attachment.isStickerAttachment() -> StickerAttachment( + attachment, + url, + session.accessToken, + cachedImagePath, + vm + ) attachment.isImageAttachment() -> ImageAttachment( attachment, url, @@ -2704,6 +2712,47 @@ private fun AttachmentView( } } +@Composable +private fun StickerAttachment( + attachment: AttachmentDto, + url: String, + token: String, + cachedImagePath: String?, + vm: QMaxViewModel +) { + LaunchedEffect(attachment.id, url) { + vm.cacheImage(attachment) + } + val model = cachedImagePath?.let(::File) ?: imageRequest(url, token) + var imageLoaded by remember(attachment.id, cachedImagePath, url) { mutableStateOf(false) } + var imageFailed by remember(attachment.id, cachedImagePath, url) { mutableStateOf(false) } + Box( + modifier = Modifier + .size(156.dp) + .clip(RoundedCornerShape(6.dp)) + .clickable { vm.openImage(cachedImagePath ?: url, listOf(cachedImagePath ?: url)) }, + contentAlignment = Alignment.Center + ) { + AsyncImage( + model = model, + contentDescription = attachment.fileName, + contentScale = ContentScale.Fit, + modifier = Modifier.fillMaxSize(), + onSuccess = { + imageLoaded = true + imageFailed = false + }, + onError = { + imageLoaded = false + imageFailed = true + } + ) + if (!imageLoaded) { + MediaImagePlaceholder(failed = imageFailed) + } + } +} + @Composable private fun EmojiAttachment( attachment: AttachmentDto, @@ -3685,7 +3734,7 @@ private fun imageRequest(url: String, token: String): ImageRequest { } private fun AttachmentDto.isImageAttachment(): Boolean { - if (isEmojiAttachment()) { + if (isEmojiAttachment() || isStickerAttachment()) { return false } return kind.equals("Image", true) || @@ -3700,6 +3749,12 @@ private fun AttachmentDto.isEmojiAttachment(): Boolean { kind.equals("Emoji", true) } +private fun AttachmentDto.isStickerAttachment(): Boolean { + val normalizedFileName = fileName.lowercase(Locale.ROOT) + return kind.equals("Sticker", true) || + normalizedFileName.startsWith("max-sticker-") +} + private fun AttachmentDto.isContactAttachment(): Boolean { return kind.equals("Contact", true) || contentType.contains("vcard", ignoreCase = true) || diff --git a/android/app/src/main/java/xyz/kusoft/qmax/core/local/AttachmentDiskCache.kt b/android/app/src/main/java/xyz/kusoft/qmax/core/local/AttachmentDiskCache.kt index ff2751f..9a2963a 100644 --- a/android/app/src/main/java/xyz/kusoft/qmax/core/local/AttachmentDiskCache.kt +++ b/android/app/src/main/java/xyz/kusoft/qmax/core/local/AttachmentDiskCache.kt @@ -81,6 +81,7 @@ class AttachmentDiskCache(context: Context) { attachment.downloadPath, attachment.fileName, attachment.contentType, + attachment.kind, attachment.fileSizeBytes.toString() ).joinToString("|") ).take(40) diff --git a/worker/src/server.js b/worker/src/server.js index 2b03ba6..88e5a26 100644 --- a/worker/src/server.js +++ b/worker/src/server.js @@ -1190,9 +1190,135 @@ async function downloadDomAudioThroughPlay(p, request) { async function downloadDomStickerThroughScreenshot(p, request) { const testId = String(request?.testId || "").trim(); const timeText = String(request?.timeText || "").trim(); - const fallbackSelector = String(request?.selector || "[data-testid^='sticker-'], button[class*='sticker' i], [class*='sticker' i] canvas, canvas"); + const fallbackSelector = String(request?.selector || "[data-testid^='sticker-'], button[class*='sticker' i], [class*='sticker' i]"); const kindLabel = String(request?.kindLabel || "MAX sticker canvas"); const selectorForTestId = (value) => `[data-testid="${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"]`; + const visualHandleFor = async (handle) => { + const visual = await handle.evaluateHandle((node) => { + const element = node instanceof Element ? node : node?.parentElement; + if (!element) return null; + + const candidates = [ + ...(element.matches("canvas,img,image,svg") ? [element] : []), + ...Array.from(element.querySelectorAll("canvas,img,image,svg,[style*='background' i]")) + ]; + const scored = candidates + .map((candidate) => { + const rect = candidate.getBoundingClientRect(); + const style = window.getComputedStyle(candidate); + if (style.visibility === "hidden" || + style.display === "none" || + rect.width <= 0 || + rect.height <= 0) { + return null; + } + + const semantic = [ + candidate.getAttribute?.("data-testid"), + candidate.getAttribute?.("aria-label"), + candidate.getAttribute?.("title"), + candidate.className?.baseVal || candidate.className, + candidate.tagName + ].filter(Boolean).join(" ").toLowerCase(); + let score = rect.width * rect.height; + if (candidate.matches("canvas,img,image,svg")) score += 100000; + if (/sticker|emoji|lottie|emoticon|\u0441\u0442\u0438\u043a\u0435\u0440|\u044d\u043c\u043e\u0434\u0437\u0438/.test(semantic)) score += 50000; + return { candidate, score }; + }) + .filter(Boolean) + .sort((a, b) => b.score - a.score); + + return scored[0]?.candidate || element; + }).catch(() => null); + + const element = visual?.asElement?.(); + if (element) { + return element; + } + + await visual?.dispose?.().catch(() => {}); + return handle; + }; + const resolveVisualHandles = async (handles) => { + const resolved = []; + for (const handle of handles) { + const visual = await visualHandleFor(handle); + if (visual !== handle) { + await handle.dispose().catch(() => {}); + } + resolved.push(visual); + } + return resolved; + }; + const captureCanvasPng = async (handle) => { + const result = await handle.evaluate(async (node) => { + const element = node instanceof Element ? node : node?.parentElement; + const canvas = element?.matches?.("canvas") + ? element + : element?.querySelector?.("canvas"); + if (!(canvas instanceof HTMLCanvasElement)) { + return { hasCanvas: false, blocked: false, dataUrl: null }; + } + + const sleep = (ms) => new Promise((resolve) => window.setTimeout(resolve, ms)); + const hasPaintedPixels = () => { + const width = canvas.width; + const height = canvas.height; + if (width <= 0 || height <= 0) return false; + + let data; + try { + const context = canvas.getContext("2d", { willReadFrequently: true }); + if (!context) return true; + data = context.getImageData(0, 0, width, height).data; + } catch (_error) { + return null; + } + + const totalPixels = width * height; + const step = Math.max(1, Math.floor(totalPixels / 12000)); + let painted = 0; + for (let pixel = 0; pixel < totalPixels; pixel += step) { + if (data[pixel * 4 + 3] > 8) { + painted++; + } + } + + return painted >= Math.max(6, Math.ceil(totalPixels / step * 0.01)); + }; + + for (let attempt = 0; attempt < 10; attempt++) { + const painted = hasPaintedPixels(); + if (painted === true) { + try { + return { hasCanvas: true, blocked: false, dataUrl: canvas.toDataURL("image/png") }; + } catch (_error) { + return { hasCanvas: true, blocked: true, dataUrl: null }; + } + } + + if (painted === null) { + try { + return { hasCanvas: true, blocked: true, dataUrl: canvas.toDataURL("image/png") }; + } catch (_error) { + return { hasCanvas: true, blocked: true, dataUrl: null }; + } + } + + await sleep(200); + } + + return { hasCanvas: true, blocked: false, dataUrl: null }; + }).catch(() => ({ hasCanvas: false, blocked: false, dataUrl: null })); + + const dataUrl = String(result?.dataUrl || ""); + const prefix = "data:image/png;base64,"; + return { + hasCanvas: Boolean(result?.hasCanvas), + blocked: Boolean(result?.blocked), + buffer: dataUrl.startsWith(prefix) ? Buffer.from(dataUrl.slice(prefix.length), "base64") : null + }; + }; const screenshotCandidate = async (handles, requestedIndex) => { const candidates = []; for (const handle of handles) { @@ -1234,11 +1360,16 @@ async function downloadDomStickerThroughScreenshot(p, request) { .join(" "); const timeMatches = requestedTimeText && (wrapperText.includes(requestedTimeText) || metaText.includes(requestedTimeText)); + const semanticRoot = element.closest("[data-testid^='sticker-'], [data-testid*='emoji' i], [class*='sticker' i], [class*='emoji' i], [class*='lottie' i]") || element; const semantic = [ element.getAttribute("data-testid"), element.getAttribute("aria-label"), element.getAttribute("title"), element.className, + semanticRoot.getAttribute?.("data-testid"), + semanticRoot.getAttribute?.("aria-label"), + semanticRoot.getAttribute?.("title"), + semanticRoot.className, wrapper.className, wrapperText ].filter(Boolean).join(" ").toLowerCase(); @@ -1328,47 +1459,59 @@ async function downloadDomStickerThroughScreenshot(p, request) { const box = await target.handle.boundingBox().catch(() => null); return box && box.width > 0 && box.height > 0 ? box : metaBox; }; - await target.handle.scrollIntoViewIfNeeded({ timeout: 3000 }).catch(() => {}); - let buffer; - if (request?.acceptSmall) { - buffer = await screenshotClip(await currentBoxOrMeta()); - if (!buffer) { - throw new Error(`${kindLabel} clip was not visible.`); + try { + await target.handle.scrollIntoViewIfNeeded({ timeout: 3000 }).catch(() => {}); + const canvasCapture = await captureCanvasPng(target.handle); + if (canvasCapture.buffer) { + return { contentType: "image/png", buffer: canvasCapture.buffer }; } - } else { - try { - buffer = await target.handle.screenshot({ type: "png", timeout: 5000 }); - } catch (error) { + + if (canvasCapture.hasCanvas && !canvasCapture.blocked) { + throw new Error(`${kindLabel} canvas is blank.`); + } + + let buffer; + if (request?.acceptSmall) { buffer = await screenshotClip(await currentBoxOrMeta()); if (!buffer) { + throw new Error(`${kindLabel} clip was not visible.`); + } + } else { + try { + buffer = await target.handle.screenshot({ type: "png", timeout: 5000 }); + } catch (error) { + buffer = await screenshotClip(await currentBoxOrMeta()); + if (!buffer) { + throw error; + } + } + } + if (!buffer) { + try { + buffer = await target.handle.screenshot({ type: "png", timeout: 5000 }); + } catch (error) { throw error; } } - } - if (!buffer) { - try { - buffer = await target.handle.screenshot({ type: "png", timeout: 5000 }); - } catch (error) { - throw error; + return { contentType: "image/png", buffer }; + } finally { + for (const item of candidates) { + await item.handle.dispose().catch(() => {}); } } - for (const item of candidates) { - await item.handle.dispose().catch(() => {}); - } - return { contentType: "image/png", buffer }; }; const index = Number.isFinite(Number(request?.index)) ? Number(request.index) : 0; if (testId) { const handles = await p.$$(selectorForTestId(testId)); - const result = await screenshotCandidate(handles, 0); + const result = await screenshotCandidate(await resolveVisualHandles(handles), 0); if (result) { return result; } } const handles = await p.$$(fallbackSelector); - const result = await screenshotCandidate(handles, index); + const result = await screenshotCandidate(await resolveVisualHandles(handles), index); if (!result) { throw new Error(`${kindLabel} was not found.`); }