Prepare local 3.4.0 web and Android release

(cherry picked from commit ca25e8becc4bfb18d63ca7920d26a984eae0e16f)

.github/workflows/draft-release.yml

@@ -154,8 +154,8 @@ jobs:
           previous=$(grep -A1 -xF "${TAG}" <<< "${tags}" | tail -1)
           echo "generating notes from ${previous}"
 
-          mise //:notes --tag "${TAG}" --previous "${previous}" > "${RUNNER_TEMP}/notes.md"
+          mise //:notes --tag "${TAG}" --previous "${previous}"
-          cat "${RUNNER_TEMP}/notes.md"
+          cat misc/release/notes.md
 
       - name: Create draft release
         env:
@@ -171,7 +171,7 @@ jobs:
             --draft \
             --verify-tag \
             --prerelease="${PRERELEASE}" \
-            --notes-file "${RUNNER_TEMP}/notes.md" \
+            --notes-file misc/release/notes.md \
             docker/docker-compose.yml \
             docker/docker-compose.rootless.yml \
             docker/example.env \

.gitignore

@@ -28,3 +28,4 @@ vite.config.js.timestamp-*
 .devcontainer/.env*
 *.tsbuildinfo
 *.tsbuildInfo
+misc/release/notes.md

mise.toml

@@ -7,6 +7,7 @@ config_roots = [
   "packages/cli",
   "deployment",
   "mobile",
+  "native",
   "e2e",
   "web",
   "docs",

mobile/ios/Runner/Images/ImageProcessing.swift

@@ -1,3 +1,4 @@
+import Accelerate
 import Foundation
 
 enum ImageProcessing {
@@ -11,4 +12,22 @@ enum ImageProcessing {
   static let cancelledResult = Result<[String: Int64]?, any Error>.success(nil)
   // Impeller shrinks each side to the GPU texture limit on its own, which squishes long images.
   static let maxPixelSize = 16384
+
+  static func rgbaBuffer(_ image: CGImage, format: vImage_CGImageFormat) throws -> vImage_Buffer {
+    guard image.bitsPerComponent == 10 else {
+      return try vImage_Buffer(cgImage: image, format: format)
+    }
+
+    // vImage can't convert packed 10-bit images.
+    let buffer = try vImage_Buffer(width: image.width, height: image.height, bitsPerPixel: format.bitsPerPixel)
+    guard let context = CGContext(data: buffer.data, width: image.width, height: image.height, bitsPerComponent: Int(format.bitsPerComponent), bytesPerRow: buffer.rowBytes, space: format.colorSpace.takeUnretainedValue(), bitmapInfo: format.bitmapInfo.rawValue) else {
+      buffer.free()
+      throw vImage.Error.invalidImageFormat
+    }
+
+    context.setRenderingIntent(format.renderingIntent)
+    context.setBlendMode(.copy)
+    context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
+    return buffer
+  }
 }

mobile/ios/Runner/Images/LocalImagesImpl.swift

@@ -165,7 +165,7 @@ class LocalImageApiImpl: LocalImageApi {
       }
 
       do {
-        let buffer = try vImage_Buffer(cgImage: cgImage, format: Self.rgbaFormat)
+        let buffer = try ImageProcessing.rgbaBuffer(cgImage, format: Self.rgbaFormat)
 
         if request.isCancelled {
           buffer.free()

mobile/ios/Runner/Images/RemoteImagesImpl.swift

@@ -148,7 +148,7 @@ class RemoteImageApiImpl: NSObject, RemoteImageApi {
       }
 
       do {
-        let buffer = try vImage_Buffer(cgImage: cgImage, format: rgbaFormat)
+        let buffer = try ImageProcessing.rgbaBuffer(cgImage, format: rgbaFormat)
 
         if request.isCancelled {
           buffer.free()

mobile/lib/pages/common/splash_screen.page.dart

@@ -299,7 +299,7 @@ class SplashScreenPageState extends ConsumerState<SplashScreenPage> {
   @override
   void initState() {
     super.initState();
-    unawaited(ref.read(authProvider.notifier).setOpenApiServiceEndpoint().whenComplete(() => resumeSession()));
+    unawaited(resumeSession());
   }
 
   Future<void> resumeSession() async {
@@ -314,13 +314,28 @@ class SplashScreenPageState extends ConsumerState<SplashScreenPage> {
       final backupNotifier = ref.read(backupProvider.notifier);
       final lifeCycle = ref.read(appStateProvider.notifier);
       final viewIntentHandler = ref.read(viewIntentHandlerProvider);
+      final authNotifier = ref.read(authProvider.notifier);
 
       unawaited(
-        ref
+        authNotifier
-            .read(authProvider.notifier)
+            .setOpenApiServiceEndpoint()
-            .saveAuthInfo(accessToken: accessToken)
+            .catchError((Object error, StackTrace stackTrace) {
+              log.warning('Failed to resolve server endpoint', error, stackTrace);
+              return null;
+            })
+            .then((_) async {
+              if (Store.tryGet(StoreKey.accessToken) != accessToken) {
+                return false;
+              }
+              await authNotifier.saveAuthInfo(accessToken: accessToken);
+              return true;
+            })
             .then(
-              (_) async {
+              (sessionResumed) async {
+                if (!sessionResumed) {
+                  return;
+                }
+
                 try {
                   wsProvider.connect();
                   unawaited(infoProvider.getServerInfo());

mobile/lib/services/api.service.dart

@@ -115,9 +115,11 @@ class ApiService {
   Future<bool> _isEndpointAvailable(String serverUrl) async {
     final endpoint = serverUrl.endsWith('/api') ? serverUrl : '$serverUrl/api';
 
+    // Probe with a throwaway client so the shared client keeps pointing at the active endpoint
+    final probeClient = ApiClient(basePath: endpoint)..client = NetworkRepository.client;
+
     try {
-      setEndpoint(endpoint);
+      await ServerApi(probeClient).pingServer().timeout(const Duration(seconds: 5));
-      await serverInfoApi.pingServer().timeout(const Duration(seconds: 5));
     } on TimeoutException catch (_) {
       return false;
     } on SocketException catch (_) {

open-api/immich-openapi-specs.json

@@ -8490,6 +8490,45 @@
         "x-immich-state": "Internal"
       }
     },
+    "/futo-backups/onboarding/restore/continue": {
+      "post": {
+        "description": "Record that the user has moved on from a finished restore, so it is no longer offered on reload.",
+        "operationId": "futoBackupsContinueAfterRestore",
+        "parameters": [],
+        "responses": {
+          "201": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "summary": "Continue after a restore",
+        "tags": [
+          "Backups"
+        ],
+        "x-immich-admin-only": true,
+        "x-immich-history": [
+          {
+            "version": "v3.4.0",
+            "state": "Added"
+          },
+          {
+            "version": "v3.4.0",
+            "state": "Internal"
+          }
+        ],
+        "x-immich-state": "Internal"
+      }
+    },
     "/futo-backups/onboarding/skip": {
       "post": {
         "description": "Skip the optional configuration steps of onboarding.",
@@ -21249,7 +21288,7 @@
   "info": {
     "title": "Immich",
     "description": "Immich API",
-    "version": "3.3.5",
+    "version": "3.4.0-rc.0",
     "contact": {}
   },
   "tags": [
@@ -28795,6 +28834,9 @@
           "requiresAuthentication": {
             "type": "boolean"
           },
+          "restoreJobId": {
+            "type": "string"
+          },
           "status": {
             "$ref": "#/components/schemas/BootstrapStatus"
           }

packages/scripts/src/cli.ts

@@ -1,5 +1,6 @@
 import { Command, Option } from 'commander';
 import {
+  FileSystem,
   GitClient,
   GitHubClient,
   type NotesOptions,
@@ -63,10 +64,11 @@ export const cli = (argv: string[]) => {
     .action(async ({ token, ...options }: NotesOptions & { token: string }) => {
       try {
         const service = new NotesService(
+          new FileSystem(),
           new GitClient(),
           new GitHubClient(token),
         );
-        process.stdout.write(await service.handleNotes(options));
+        await service.handleNotes(options);
       } catch (error) {
         console.error(error instanceof Error ? error.message : error);
         process.exit(1);

packages/scripts/src/commands/notes.spec.ts

@@ -2,6 +2,7 @@ import type { Public } from 'src/types.js';
 import { beforeEach, describe, expect, it, type Mocked, vi } from 'vitest';
 import {
   type Contributor,
+  type FileSystem,
   type GitClient,
   type GitHubClient,
   NotesService,
@@ -37,6 +38,7 @@ const changelog = (...lines: string[]) =>
 const body = (notes: string) => notes.slice(notes.indexOf("## What's Changed"));
 
 describe(NotesService.name, () => {
+  let fs: Mocked<Public<FileSystem>>;
   let git: Mocked<Public<GitClient>>;
   let github: Mocked<Public<GitHubClient>>;
   let sut: NotesService;
@@ -88,12 +90,16 @@ describe(NotesService.name, () => {
     github.getFirstTimers.mockResolvedValue(firstTimers);
   };
 
-  const run = () => sut.handleNotes({ tag: 'v3.3.0', previous: 'v3.2.4' });
+  const run = async () => {
+    await sut.handleNotes({ tag: 'v3.3.0', previous: 'v3.2.4' });
+    return fs.writeFileSync.mock.lastCall?.[1] as string;
+  };
 
   beforeEach(() => {
+    fs = { writeFileSync: vi.fn() };
     git = { log: vi.fn() };
     github = { getPullRequests: vi.fn(), getFirstTimers: vi.fn() };
-    sut = new NotesService(git, github);
+    sut = new NotesService(fs, git, github);
   });
 
   it('should compare the previous tag with the new tag', async () => {
@@ -104,6 +110,18 @@ describe(NotesService.name, () => {
     expect(git.log).toHaveBeenCalledWith('v3.2.4', 'v3.3.0');
   });
 
+  it('should write the notes to misc/release/notes.md', async () => {
+    setup([{ number: 1 }]);
+
+    await run();
+
+    expect(fs.writeFileSync).toHaveBeenCalledOnce();
+    expect(fs.writeFileSync).toHaveBeenCalledWith(
+      expect.stringMatching(/\/misc\/release\/notes\.md$/),
+      expect.any(String),
+    );
+  });
+
   it('should start with the template and end with the full changelog', async () => {
     setup([{ number: 1 }]);
 

packages/scripts/src/commands/notes.ts

@@ -1,5 +1,5 @@
 import { execFileSync } from 'node:child_process';
-import { existsSync, readFileSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import type { Public } from 'src/types.js';
@@ -15,6 +15,7 @@ const REPO = 'immich-app/immich';
 const Files = {
   Config: join(root, '.github/release.yml'),
   Template: join(root, 'misc/release/notes.tmpl'),
+  Output: join(root, 'misc/release/notes.md'),
 };
 
 export type NotesOptions = {
@@ -55,6 +56,12 @@ type PullRequestNode = {
 const PULL_REQUEST_BATCH = 50;
 const SEARCH_BATCH = 20;
 
+export class FileSystem {
+  writeFileSync(path: string, data: string) {
+    writeFileSync(path, data);
+  }
+}
+
 export class GitClient {
   log(previous: string, tag: string) {
     return execFileSync(
@@ -189,6 +196,7 @@ export class GitHubClient {
 
 export class NotesService {
   constructor(
+    private fs: Public<FileSystem>,
     private git: Public<GitClient>,
     private github: Public<GitHubClient>,
   ) {}
@@ -198,8 +206,9 @@ export class NotesService {
     const pulls = await this.getPullRequests(commits);
     const changes = this.selectChanges(commits, pulls);
     const firstTimers = await this.getFirstTimers(changes);
+    const notes = this.render({ tag, previous, changes, firstTimers });
 
-    return this.render({ tag, previous, changes, firstTimers });
+    this.fs.writeFileSync(Files.Output, notes);
   }
 
   private parseCommits(log: string): Commit[] {

packages/sdk/src/fetch-client.ts

@@ -1,6 +1,6 @@
 /**
  * Immich
- * 3.3.3
+ * 3.4.0-rc.0
  * DO NOT MODIFY - This file has been generated using oazapfts.
  * See https://www.npmjs.com/package/oazapfts
  */
@@ -2006,6 +2006,7 @@ export type OnboardingStatusResponseDto = {
     hasTelemetry: TelemetryLevel;
     isAuthenticated: boolean;
     requiresAuthentication: boolean;
+    restoreJobId?: string;
     status: BootstrapStatus;
 };
 export type CurrentRecoveryKeyResponse = {
@@ -6297,6 +6298,15 @@ export function futoBackupsReportError(opts?: Oazapfts.RequestOpts) {
         method: "POST"
     }));
 }
+/**
+ * Continue after a restore
+ */
+export function futoBackupsContinueAfterRestore(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchText("/futo-backups/onboarding/restore/continue", {
+        ...opts,
+        method: "POST"
+    }));
+}
 /**
  * Skip optional backup onboarding
  */

pnpm-lock.yaml

@@ -566,8 +566,8 @@ importers:
         specifier: 2.0.0-rc13
         version: 2.0.0-rc13
       '@futo-org/backups-orchestrator-api':
-        specifier: 1.11.5
+        specifier: 1.12.1
-        version: 1.11.5(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0)(@nestjs/platform-socket.io@12.1.0)(@nestjs/schedule@12.0.2(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0))(@nestjs/swagger@12.0.2(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0)(@typescript/typescript6@6.0.2)(reflect-metadata@0.2.2))(@nestjs/websockets@12.1.0)(reflect-metadata@0.2.2)(supports-color@8.1.1)
+        version: 1.12.1(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0)(@nestjs/platform-socket.io@12.1.0)(@nestjs/schedule@12.0.2(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0))(@nestjs/swagger@12.0.2(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0)(@typescript/typescript6@6.0.2)(reflect-metadata@0.2.2))(@nestjs/websockets@12.1.0)(reflect-metadata@0.2.2)(supports-color@8.1.1)
       '@immich/plugin-sdk':
         specifier: workspace:*
         version: link:../packages/plugin-sdk
@@ -957,8 +957,8 @@ importers:
         specifier: ^3.0.0
         version: 3.5.20
       '@futo-org/backups-orchestrator-ui':
-        specifier: 1.11.5
+        specifier: 1.12.1
-        version: 1.11.5(@immich/ui@0.91.1(@sveltejs/kit@3.1.0(@opentelemetry/api@1.10.0)(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(@typescript/typescript6@6.0.2)(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(svelte@5.57.2(@typescript-eslint/types@8.70.1))(zod@4.4.3))(@lingui/cli@6.11.0(esbuild@0.28.2)(rolldown@1.2.9)(supports-color@8.1.1))(@typescript/typescript6@6.0.2)(supports-color@8.1.1)(svelte@5.57.2(@typescript-eslint/types@8.70.1))
+        version: 1.12.1(@immich/ui@0.91.1(@sveltejs/kit@3.1.0(@opentelemetry/api@1.10.0)(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(@typescript/typescript6@6.0.2)(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(svelte@5.57.2(@typescript-eslint/types@8.70.1))(zod@4.4.3))(@lingui/cli@6.11.0(esbuild@0.28.2)(rolldown@1.2.9)(supports-color@8.1.1))(@typescript/typescript6@6.0.2)(supports-color@8.1.1)(svelte@5.57.2(@typescript-eslint/types@8.70.1))
       '@immich/justified-layout-wasm':
         specifier: ^0.4.3
         version: 0.4.3
@@ -2944,8 +2944,8 @@ packages:
   '@futo-org/backups-api-client@1.12.1':
     resolution: {integrity: sha512-y1AtOftdZnFpoU7YRteD6U0qip4zWOg/JuiplGGGotdBnNMipOCSDj7ttU2huWRIEfZRosgAPiUi/lt/Bns/Xg==}
 
-  '@futo-org/backups-orchestrator-api@1.11.5':
+  '@futo-org/backups-orchestrator-api@1.12.1':
-    resolution: {integrity: sha512-YcIdWrgoyYx8L1vu2Gwtb+k1lQL32xQOK65JAAYMdZ7uEb+B/uEwHQOJWgCoxH6exwvEVtWPbvj3Mkn84r5pTQ==}
+    resolution: {integrity: sha512-qTdKjndwxRmbykJswtbLjSpZ1is3/jAPgdqWeKJ4Kmnc9ZnFjn6tFb3HL4NDztCNIcaSWuLMIDJQpGWcJ3kdAA==}
     peerDependencies:
       '@nestjs/common': ^12.0.0
       '@nestjs/core': ^12.0.0
@@ -2954,8 +2954,8 @@ packages:
       '@nestjs/swagger': ^12.0.0
       '@nestjs/websockets': ^12.0.0
 
-  '@futo-org/backups-orchestrator-ui@1.11.5':
+  '@futo-org/backups-orchestrator-ui@1.12.1':
-    resolution: {integrity: sha512-sfUfsJCeR7HEfmbLu/eAACAtY27ALxLPlZ9BYW9MnnQ9BXQHGBnM+hZ0pGs/YwtYuNsAeoq5kEbIadVN5sX6Gw==}
+    resolution: {integrity: sha512-GejBWdU3w3dnVfK5llQVn/r6vGLrcZ/0ozzBnFlM5YdiCPr9NJebFcL9WMvghxjvw+CPTfDmYQfVbm35qFAfmA==}
     peerDependencies:
       '@immich/ui': ^0.90.0
       svelte: ^5.0.0
@@ -16341,7 +16341,7 @@ snapshots:
     dependencies:
       '@oazapfts/runtime': 1.2.0
 
-  '@futo-org/backups-orchestrator-api@1.11.5(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0)(@nestjs/platform-socket.io@12.1.0)(@nestjs/schedule@12.0.2(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0))(@nestjs/swagger@12.0.2(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0)(@typescript/typescript6@6.0.2)(reflect-metadata@0.2.2))(@nestjs/websockets@12.1.0)(reflect-metadata@0.2.2)(supports-color@8.1.1)':
+  '@futo-org/backups-orchestrator-api@1.12.1(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0)(@nestjs/platform-socket.io@12.1.0)(@nestjs/schedule@12.0.2(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0))(@nestjs/swagger@12.0.2(@nestjs/common@12.1.0(reflect-metadata@0.2.2)(rxjs@7.8.2)(supports-color@8.1.1))(@nestjs/core@12.1.0)(@typescript/typescript6@6.0.2)(reflect-metadata@0.2.2))(@nestjs/websockets@12.1.0)(reflect-metadata@0.2.2)(supports-color@8.1.1)':
     dependencies:
       '@futo-org/backups-api-client': 1.12.1
       '@futo-org/restic-wrapper': 1.7.0
@@ -16372,7 +16372,7 @@ snapshots:
       - supports-color
       - utf-8-validate
 
-  '@futo-org/backups-orchestrator-ui@1.11.5(@immich/ui@0.91.1(@sveltejs/kit@3.1.0(@opentelemetry/api@1.10.0)(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(@typescript/typescript6@6.0.2)(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(svelte@5.57.2(@typescript-eslint/types@8.70.1))(zod@4.4.3))(@lingui/cli@6.11.0(esbuild@0.28.2)(rolldown@1.2.9)(supports-color@8.1.1))(@typescript/typescript6@6.0.2)(supports-color@8.1.1)(svelte@5.57.2(@typescript-eslint/types@8.70.1))':
+  '@futo-org/backups-orchestrator-ui@1.12.1(@immich/ui@0.91.1(@sveltejs/kit@3.1.0(@opentelemetry/api@1.10.0)(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(@typescript/typescript6@6.0.2)(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(svelte@5.57.2(@typescript-eslint/types@8.70.1))(zod@4.4.3))(@lingui/cli@6.11.0(esbuild@0.28.2)(rolldown@1.2.9)(supports-color@8.1.1))(@typescript/typescript6@6.0.2)(supports-color@8.1.1)(svelte@5.57.2(@typescript-eslint/types@8.70.1))':
     dependencies:
       '@futo-org/backups-api-client': 1.12.1
       '@immich/ui': 0.91.1(@sveltejs/kit@3.1.0(@opentelemetry/api@1.10.0)(@sveltejs/vite-plugin-svelte@7.3.1(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(@typescript/typescript6@6.0.2)(svelte@5.57.2(@typescript-eslint/types@8.70.1))(vite@8.3.0(@types/node@24.19.2)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.102.0)(terser@5.49.0)(tsx@4.23.15)(yaml@2.9.1)))(svelte@5.57.2(@typescript-eslint/types@8.70.1))(zod@4.4.3)

server/Dockerfile

@@ -119,7 +119,7 @@ ENV IMMICH_SOURCE_COMMIT=${BUILD_SOURCE_COMMIT}
 ENV IMMICH_SOURCE_URL=https://github.com/immich-app/immich/commit/${BUILD_SOURCE_COMMIT}
 
 COPY --from=ghcr.io/futo-org/backups/restic-proxy:v1.11.2@sha256:a88252d884d22a55749f45dd39a2458c6391ca3e4d3b0d9e0c2246786cf923be /usr/local/bin/restic-proxy /usr/local/bin/restic-proxy
-COPY --from=ghcr.io/futo-org/restic:sha-a24cc22@sha256:7b5cdba9cc8690ecc528e239e6f8b2cfb9ef3644895cafd0e0fa9003bcded1f6 /usr/bin/restic /usr/local/bin/restic
+COPY --from=ghcr.io/futo-org/restic:sha-1e904f0@sha256:501bb3e696bd78db0d89ae4e5c51b6993b9cb3f88e657020777dba49226169bc /usr/bin/restic /usr/local/bin/restic
 
 VOLUME /data
 EXPOSE 2283

server/package.json

@@ -40,7 +40,7 @@
   },
   "dependencies": {
     "@extism/extism": "2.0.0-rc13",
-    "@futo-org/backups-orchestrator-api": "1.11.5",
+    "@futo-org/backups-orchestrator-api": "1.12.1",
     "@immich/plugin-sdk": "workspace:*",
     "@immich/sql-tools": "^0.6.6",
     "@nestjs/bullmq": "^12.0.0",

server/src/queries/asset.repository.sql

@@ -511,18 +511,22 @@ limit
   $4
 
 -- AssetRepository.detectOfflineExternalAssets
-update "asset"
+begin
-set
+declare offline_check no scroll cursor for (
-  "isOffline" = $1,
+  select
-  "deletedAt" = $2
+    "asset"."id",
-where
+    "asset"."originalPath"
-  "isOffline" = $3
+  from
-  and "isExternal" = $4
+    "asset"
-  and "libraryId" = $5::uuid
+  where
-  and (
+    "asset"."libraryId" = $1::uuid
-    not "originalPath" like $6
+    and "asset"."isOffline" = $2
-    or "originalPath" ~* $7
+    and "asset"."isExternal" = $3
-  )
+)
+fetch 10000
+from
+  offline_check
+commit
 
 -- AssetRepository.filterNewExternalAssetPaths
 select

server/src/repositories/asset.repository.ts

@@ -7,13 +7,13 @@ import {
   type SelectQueryBuilder,
   type Selectable,
   type ShallowDehydrateObject,
-  UpdateResult,
   type Updateable,
   sql,
 } from 'kysely';
 import { jsonArrayFrom } from 'kysely/helpers/postgres';
 import { isEmpty, isUndefined, omitBy } from 'lodash-es';
 import { InjectKysely } from 'nestjs-kysely';
+import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants.js';
 import { LockableProperty, Stack } from 'src/database.js';
 import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators.js';
 import { type AuthDto, isMetadataHidden } from 'src/dtos/auth.dto.js';
@@ -54,7 +54,6 @@ import {
   withTagId,
   withTags,
 } from 'src/utils/database.js';
-import { globToPostgresRegex } from 'src/utils/misc.js';
 
 export type AssetStats = Record<AssetType, number>;
 
@@ -698,13 +697,14 @@ export class AssetRepository {
   }
 
   findLivePhotoMatch(options: LivePhotoSearchOptions) {
-    const { ownerId, otherAssetId, livePhotoCID, type } = options;
+    const { ownerId, libraryId, otherAssetId, livePhotoCID, type } = options;
     return this.db
       .selectFrom('asset')
       .select(['asset.id', 'asset.ownerId'])
       .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
       .where('id', '!=', asUuid(otherAssetId))
       .where('ownerId', '=', asUuid(ownerId))
+      .$call((qb) => (libraryId ? qb.where('libraryId', '=', asUuid(libraryId)) : qb.where('libraryId', 'is', null)))
       .where('type', '=', type)
       .where('asset_exif.livePhotoCID', '=', livePhotoCID)
       .limit(1)
@@ -1091,31 +1091,45 @@ export class AssetRepository {
       .execute();
   }
 
-  @GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING], [DummyValue.STRING]] })
+  @GenerateSql({ params: [DummyValue.UUID, () => true] })
-  async detectOfflineExternalAssets(
+  detectOfflineExternalAssets(libraryId: string, isInLibrary: (originalPath: string) => boolean): Promise<number> {
-    libraryId: string,
+    // read and write on one connection, so concurrent library scans can't deadlock on the pool
-    importPaths: string[],
+    return this.db.transaction().execute(async (trx) => {
-    exclusionPatterns: string[],
+      const query = trx
-  ): Promise<UpdateResult> {
+        .selectFrom('asset')
-    const paths = importPaths.map((importPath) => `${importPath}%`);
+        .select(['asset.id', 'asset.originalPath'])
-    const exclusions = exclusionPatterns.map((pattern) => globToPostgresRegex(pattern));
+        .where('asset.libraryId', '=', asUuid(libraryId))
+        .where('asset.isOffline', '=', false)
+        .where('asset.isExternal', '=', true);
+      await sql`declare offline_check no scroll cursor for ${query}`.execute(trx);
 
-    return this.db
+      const fetchBatch = async () => {
-      .updateTable('asset')
+        const { rows } = await sql<{ id: string; originalPath: string }>`
-      .set({
+          fetch ${sql.lit(JOBS_LIBRARY_PAGINATION_SIZE)} from offline_check
-        isOffline: true,
+        `.execute(trx);
-        deletedAt: new Date(),
+        return rows;
-      })
+      };
-      .where('isOffline', '=', false)
+
-      .where('isExternal', '=', true)
+      const deletedAt = new Date();
-      .where('libraryId', '=', asUuid(libraryId))
+      let count = 0;
-      .where((eb) =>
+      for (let rows = await fetchBatch(); rows.length > 0; rows = await fetchBatch()) {
-        eb.or([
+        const ids = rows.filter(({ originalPath }) => !isInLibrary(originalPath)).map(({ id }) => id);
-          eb.not(eb.or(paths.map((path) => eb('originalPath', 'like', path)))),
+        if (ids.length === 0) {
-          eb.or(exclusions.map((pattern) => eb('originalPath', '~*', pattern))),
+          continue;
-        ]),
+        }
-      )
+
-      .executeTakeFirstOrThrow();
+        // a concurrent check of the same library may have offlined these rows after the cursor was declared
+        const { numUpdatedRows } = await trx
+          .updateTable('asset')
+          .set({ isOffline: true, deletedAt })
+          .where('id', '=', anyUuid(ids))
+          .where('isOffline', '=', false)
+          .executeTakeFirst();
+        count += Number(numUpdatedRows);
+      }
+
+      return count;
+    });
   }
 
   @GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] })

server/src/services/database-backup.service.spec.ts

@@ -241,11 +241,19 @@ describe(DatabaseBackupService.name, () => {
       expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith('gzip', ['--rsyncable']);
     });
 
-    it('should compress with pigz on every available core when it is installed', async () => {
+    it.each([
+      { cores: 1, processes: '1' },
+      { cores: 8, processes: '6' },
+      { cores: 72, processes: '32' },
+    ])('should compress with pigz on $processes of $cores cores when it is installed', async ({ cores, processes }) => {
       mocks.process.isCommandAvailable.mockResolvedValue(true);
-      mocks.process.availableParallelism.mockReturnValue(8);
+      mocks.process.availableParallelism.mockReturnValue(cores);
       await sut.handleBackupDatabase();
-      expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith('pigz', ['--rsyncable', '--processes', '8']);
+      expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith('pigz', [
+        '--rsyncable',
+        '--processes',
+        processes,
+      ]);
     });
 
     it('should fail if pg_dump fails', async () => {

server/src/services/database-backup.service.ts

@@ -271,7 +271,7 @@ export class DatabaseBackupService {
 
   private async getCompressionCommand() {
     if (await this.processRepository.isCommandAvailable('pigz')) {
-      const processes = this.processRepository.availableParallelism();
+      const processes = Math.min(Math.max(Math.floor(this.processRepository.availableParallelism() * 0.75), 1), 32);
       return { command: 'pigz', args: ['--rsyncable', '--processes', processes.toString()] };
     }
 

server/src/services/library.service.spec.ts

@@ -273,31 +273,42 @@ describe(LibraryService.name, () => {
   });
 
   describe('handleQueueSyncAssets', () => {
-    it('should call the offline check', async () => {
+    it('should offline assets outside the import paths or covered by an exclusion pattern', async () => {
-      const library = factory.library();
+      const library = factory.library({ importPaths: ['/import/'], exclusionPatterns: ['**/raw/**'] });
+      const asset = AssetFactory.create({ libraryId: library.id, originalPath: '/import/photo.jpg' });
+
+      mocks.library.get.mockResolvedValue(library);
+      mocks.asset.getLibraryAssetCount.mockResolvedValue(3);
+      mocks.asset.detectOfflineExternalAssets.mockResolvedValue(2);
+      mocks.library.streamAssetIds.mockReturnValue(makeStream([asset]));
+
+      await expect(sut.handleQueueSyncAssets({ id: library.id })).resolves.toBe(JobStatus.Success);
+
+      expect(mocks.asset.detectOfflineExternalAssets).toHaveBeenCalledWith(library.id, expect.any(Function));
+      const [, isInLibrary] = mocks.asset.detectOfflineExternalAssets.mock.calls[0];
+      expect(isInLibrary('/import/photo.jpg')).toBe(true);
+      expect(isInLibrary('/elsewhere/photo.jpg')).toBe(false);
+      expect(isInLibrary('/import/RAW/photo.jpg')).toBe(false);
+    });
+
+    it('should not queue asset sync when every asset was offlined', async () => {
+      const library = factory.library({ importPaths: ['/import/'] });
 
       mocks.library.get.mockResolvedValue(library);
-      mocks.storage.walk.mockImplementation(async function* generator() {});
       mocks.asset.getLibraryAssetCount.mockResolvedValue(1);
-      mocks.asset.detectOfflineExternalAssets.mockResolvedValue({ numUpdatedRows: 1n });
+      mocks.asset.detectOfflineExternalAssets.mockResolvedValue(1);
 
-      const response = await sut.handleQueueSyncAssets({ id: library.id });
+      await expect(sut.handleQueueSyncAssets({ id: library.id })).resolves.toBe(JobStatus.Success);
 
-      expect(response).toBe(JobStatus.Success);
+      expect(mocks.library.streamAssetIds).not.toHaveBeenCalled();
-      expect(mocks.asset.detectOfflineExternalAssets).toHaveBeenCalledWith(
+      expect(mocks.job.queue).not.toHaveBeenCalled();
-        library.id,
-        library.importPaths,
-        library.exclusionPatterns,
-      );
     });
 
     it('should skip an empty library', async () => {
       const library = factory.library();
 
       mocks.library.get.mockResolvedValue(library);
-      mocks.storage.walk.mockImplementation(async function* generator() {});
       mocks.asset.getLibraryAssetCount.mockResolvedValue(0);
-      mocks.asset.detectOfflineExternalAssets.mockResolvedValue({ numUpdatedRows: 1n });
 
       const response = await sut.handleQueueSyncAssets({ id: library.id });
 
@@ -310,10 +321,9 @@ describe(LibraryService.name, () => {
       const asset = AssetFactory.create({ libraryId: library.id, isExternal: true });
 
       mocks.library.get.mockResolvedValue(library);
-      mocks.storage.walk.mockImplementation(async function* generator() {});
+      mocks.asset.detectOfflineExternalAssets.mockResolvedValue(0);
       mocks.library.streamAssetIds.mockReturnValue(makeStream([asset]));
       mocks.asset.getLibraryAssetCount.mockResolvedValue(1);
-      mocks.asset.detectOfflineExternalAssets.mockResolvedValue({ numUpdatedRows: 0n });
 
       const response = await sut.handleQueueSyncAssets({ id: library.id });
 
@@ -330,11 +340,6 @@ describe(LibraryService.name, () => {
       });
 
       expect(response).toBe(JobStatus.Success);
-      expect(mocks.asset.detectOfflineExternalAssets).toHaveBeenCalledWith(
-        library.id,
-        library.importPaths,
-        library.exclusionPatterns,
-      );
     });
 
     it("should fail if library can't be found", async () => {

server/src/services/library.service.ts

@@ -34,7 +34,7 @@ import { AssetSyncResult } from 'src/repositories/library.repository.js';
 import { AssetTable } from 'src/schema/tables/asset.table.js';
 import { BaseService } from 'src/services/base.service.js';
 import { mimeTypes } from 'src/utils/mime-types.js';
-import { batched, findOrFail, handlePromiseError } from 'src/utils/misc.js';
+import { batched, createLibraryMatcher, findOrFail, handlePromiseError } from 'src/utils/misc.js';
 
 @Injectable()
 export class LibraryService extends BaseService {
@@ -483,6 +483,7 @@ export class LibraryService extends BaseService {
   @OnJob({ name: JobName.LibrarySyncAssets, queue: QueueName.Library })
   async handleSyncAssets(job: JobOf<JobName.LibrarySyncAssets>): Promise<JobStatus> {
     const assets = await this.assetJobRepository.getForSyncAssets(job.assetIds);
+    const isInLibrary = createLibraryMatcher(job);
 
     const assetIdsToOffline: string[] = [];
     const trashedAssetIdsToOffline: string[] = [];
@@ -517,33 +518,19 @@ export class LibraryService extends BaseService {
           break;
         }
         case AssetSyncResult.CHECK_OFFLINE: {
-          const isInImportPath = job.importPaths.some((path) => asset.originalPath.startsWith(path));
+          if (!isInLibrary(asset.originalPath)) {
-
-          if (!isInImportPath) {
             this.logger.verbose(
-              `Offline asset ${asset.originalPath} is still not in any import path, keeping offline in library ${job.libraryId}`,
+              `Offline asset ${asset.originalPath} is still not in any import path or covered by an exclusion pattern, keeping offline in library ${job.libraryId}`,
             );
             break;
           }
 
-          const isExcluded = job.exclusionPatterns.some((pattern) =>
+          this.logger.debug(`Offline asset ${asset.originalPath} is now online in library ${job.libraryId}`);
-            picomatch.isMatch(asset.originalPath, pattern, { nocase: true }),
+          if (asset.status === AssetStatus.Trashed) {
-          );
+            trashedAssetIdsToOnline.push(asset.id);
-
+          } else {
-          if (!isExcluded) {
+            assetIdsToOnline.push(asset.id);
-            this.logger.debug(`Offline asset ${asset.originalPath} is now online in library ${job.libraryId}`);
-            if (asset.status === AssetStatus.Trashed) {
-              trashedAssetIdsToOnline.push(asset.id);
-            } else {
-              assetIdsToOnline.push(asset.id);
-            }
-            break;
           }
-
-          this.logger.verbose(
-            `Offline asset ${asset.originalPath} is in an import path but still covered by exclusion pattern, keeping offline in library ${job.libraryId}`,
-          );
-
           break;
         }
       }
@@ -721,14 +708,11 @@ export class LibraryService extends BaseService {
       `Checking ${assetCount} asset(s) against import paths and exclusion patterns in library ${library.id}...`,
     );
 
-    const offlineResult = await this.assetRepository.detectOfflineExternalAssets(
+    const affectedAssetCount = await this.assetRepository.detectOfflineExternalAssets(
       library.id,
-      library.importPaths,
+      createLibraryMatcher(library),
-      library.exclusionPatterns,
     );
 
-    const affectedAssetCount = Number(offlineResult.numUpdatedRows);
-
     this.logger.log(
       `${affectedAssetCount} asset(s) out of ${assetCount} were offlined due to import paths and/or exclusion pattern(s) in library ${library.id}`,
     );

server/src/services/media.service.spec.ts

@@ -3099,7 +3099,7 @@ describe(MediaService.name, () => {
             '-v',
             'verbose',
             '-vf',
-            'hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq:format=nv12',
+            'format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq:format=nv12',
             '-preset',
             '7',
             '-global_quality:v',
@@ -3116,6 +3116,24 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should tone-map to nv12 before upload for sw decode qsv', async () => {
+      mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
+      mocks.systemMetadata.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, accelDecode: false },
+      });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          outputOptions: expect.arrayContaining([
+            '-vf',
+            'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=nv12,hwupload=extra_hw_frames=64',
+          ]),
+        }),
+      );
+    });
+
     it('should set options for qsv with custom dri node with sw decode', async () => {
       mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
       mocks.systemMetadata.get.mockResolvedValue({
@@ -3347,7 +3365,7 @@ describe(MediaService.name, () => {
             '-v',
             'verbose',
             '-vf',
-            'hwupload=extra_hw_frames=64,scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12',
+            'format=nv12,hwupload=extra_hw_frames=64,scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12',
             '-compression_level',
             '7',
             '-rc_mode',
@@ -3358,6 +3376,39 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should convert to nv12 before upload for sw decode vaapi without scaling', async () => {
+      mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
+      mocks.systemMetadata.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: false, targetResolution: 'original' },
+      });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          outputOptions: expect.arrayContaining(['-vf', 'format=nv12,hwupload=extra_hw_frames=64']),
+        }),
+      );
+    });
+
+    it('should tone-map to nv12 before upload for sw decode vaapi', async () => {
+      mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
+      mocks.systemMetadata.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: false },
+      });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          outputOptions: expect.arrayContaining([
+            '-vf',
+            'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=nv12,hwupload=extra_hw_frames=64',
+          ]),
+        }),
+      );
+    });
+
     it('should set vbr options for vaapi when max bitrate is enabled', async () => {
       mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
       mocks.systemMetadata.get.mockResolvedValue({

server/src/services/metadata.service.spec.ts

@@ -2,6 +2,7 @@ import { BinaryField, ExifDateTime } from 'exiftool-vendored';
 import { DateTime } from 'luxon';
 import { randomBytes } from 'node:crypto';
 import { Stats } from 'node:fs';
+import { StorageCore, ThumbnailPathEntity } from 'src/cores/storage.core.js';
 import { defaults } from 'src/dtos/config.dto.js';
 import {
   AssetFileType,
@@ -13,6 +14,7 @@ import {
   JobName,
   JobStatus,
   SourceType,
+  StorageFolder,
 } from 'src/enum.js';
 import { ImmichTags } from 'src/repositories/metadata.repository.js';
 import { MetadataService, firstDateTime } from 'src/services/metadata.service.js';
@@ -21,7 +23,7 @@ import { PersonGroupFactory } from 'test/factories/person-group.factory.js';
 import { PersonFactory } from 'test/factories/person.factory.js';
 import { videoInfoStub } from 'test/fixtures/media.stub.js';
 import { tagStub } from 'test/fixtures/tag.stub.js';
-import { getForMetadataExtraction, getForSidecarWrite } from 'test/mappers.js';
+import { getForAsset, getForMetadataExtraction, getForSidecarWrite } from 'test/mappers.js';
 import { factory } from 'test/small.factory.js';
 import { ServiceMocks, makeStream, newTestService } from 'test/utils.js';
 
@@ -953,27 +955,47 @@ describe(MetadataService.name, () => {
       });
     });
 
-    it('should delete old motion photo video assets if they do not match what is extracted', async () => {
+    it.each([
-      const motionAsset = AssetFactory.create({ type: AssetType.Video, visibility: AssetVisibility.Hidden });
+      {
-      const asset = AssetFactory.create({ livePhotoVideoId: motionAsset.id });
+        motion: 'an extracted motion video',
-      mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
+        getOriginalPath: (motion: ThumbnailPathEntity) => StorageCore.getAndroidMotionPath(motion, motion.id),
-      mockReadTags({
+        deleteOnDisk: true,
-        Directory: 'foo/bar/',
+      },
-        MotionPhoto: 1,
+      {
-        MicroVideo: 1,
+        motion: 'an uploaded motion video',
-        MicroVideoOffset: 1,
+        getOriginalPath: (motion: ThumbnailPathEntity) =>
-      });
+          StorageCore.getNestedPath(StorageFolder.Upload, motion.ownerId, `${motion.id}.mov`),
-      mocks.crypto.hashSha1.mockReturnValue(randomBytes(512));
+        deleteOnDisk: false,
-      mocks.asset.create.mockResolvedValue(AssetFactory.create({ type: AssetType.Video }));
+      },
-      const video = randomBytes(512);
+      { motion: 'an external motion video', getOriginalPath: () => '/mnt/photos/IMG_0001.mov', deleteOnDisk: false },
-      mocks.storage.readFile.mockResolvedValue(video);
+      { motion: 'a missing motion video', getOriginalPath: undefined, deleteOnDisk: false },
+    ])(
+      'should delete $motion that does not match what is extracted (deleteOnDisk: $deleteOnDisk)',
+      async ({ getOriginalPath, deleteOnDisk }) => {
+        const motionAsset = AssetFactory.create({ type: AssetType.Video, visibility: AssetVisibility.Hidden });
+        const asset = AssetFactory.create({ livePhotoVideoId: motionAsset.id });
+        const oldMotion =
+          getOriginalPath && getForAsset({ ...motionAsset, originalPath: getOriginalPath(motionAsset) });
+        mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
+        mocks.asset.getById.mockImplementation((id) => Promise.resolve(id === motionAsset.id ? oldMotion : undefined));
+        mockReadTags({
+          Directory: 'foo/bar/',
+          MotionPhoto: 1,
+          MicroVideo: 1,
+          MicroVideoOffset: 1,
+        });
+        mocks.crypto.hashSha1.mockReturnValue(randomBytes(512));
+        mocks.asset.create.mockResolvedValue(AssetFactory.create({ type: AssetType.Video }));
+        const video = randomBytes(512);
+        mocks.storage.readFile.mockResolvedValue(video);
 
-      await sut.handleMetadataExtraction({ id: asset.id });
+        await sut.handleMetadataExtraction({ id: asset.id });
-      expect(mocks.job.queue).toHaveBeenNthCalledWith(1, {
+        expect(mocks.job.queue).toHaveBeenNthCalledWith(1, {
-        name: JobName.AssetDelete,
+          name: JobName.AssetDelete,
-        data: { id: asset.livePhotoVideoId, deleteOnDisk: true },
+          data: { id: motionAsset.id, deleteOnDisk },
-      });
+        });
-    });
+      },
+    );
 
     it('should not create a new motion photo video asset if the hash of the extracted video matches an existing asset', async () => {
       const motionAsset = AssetFactory.create({ type: AssetType.Video, visibility: AssetVisibility.Hidden });

server/src/services/metadata.service.ts

@@ -789,9 +789,12 @@ export class MetadataService extends BaseService {
         // (if it did, getByChecksum() would've returned a motionAsset with the same ID as livePhotoVideoId)
         // note asset.livePhotoVideoId is not motionAsset.id yet
         if (asset.livePhotoVideoId) {
+          // the old motion may be the user's own file (linked by livePhotoCID or manually), only remove ones Immich extracted
+          const oldMotion = await this.assetRepository.getById(asset.livePhotoVideoId);
+          const deleteOnDisk = !!oldMotion && StorageCore.isAndroidMotionPath(oldMotion.originalPath);
           await this.jobRepository.queue({
             name: JobName.AssetDelete,
-            data: { id: asset.livePhotoVideoId, deleteOnDisk: true },
+            data: { id: asset.livePhotoVideoId, deleteOnDisk },
           });
           this.logger.log(`Removed old motion photo video asset (${asset.livePhotoVideoId})`);
         }

server/src/utils/media.ts

@@ -399,13 +399,13 @@ export class BaseConfig implements VideoCodecSWConfig {
     };
   }
 
-  getToneMapping(videoStream: VideoStreamInfo) {
+  getToneMapping(videoStream: VideoStreamInfo, format = 'yuv420p') {
     if (!this.shouldToneMap(videoStream)) {
       return [];
     }
 
     const { primaries, transfer, matrix } = this.getColors();
-    const options = `tonemapx=tonemap=${this.config.tonemap}:desat=0:p=${primaries}:t=${transfer}:m=${matrix}:r=pc:peak=100:format=yuv420p`;
+    const options = `tonemapx=tonemap=${this.config.tonemap}:desat=0:p=${primaries}:t=${transfer}:m=${matrix}:r=pc:peak=100:format=${format}`;
     return [options];
   }
 
@@ -818,6 +818,15 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
     return options;
   }
 
+  getToneMapping(videoStream: VideoStreamInfo) {
+    // Output nv12 when tone-mapping instead of having ffmpeg convert from yuv420p
+    const options = super.getToneMapping(videoStream, 'nv12');
+    if (options.length === 0 && videoStream.pixelFormat !== 'nv12') {
+      options.push('format=nv12');
+    }
+    return options;
+  }
+
   getPresetOptions() {
     let presetIndex = this.getPresetIndex();
     if (presetIndex < 0) {
@@ -952,6 +961,15 @@ export class VaapiSwDecodeConfig extends BaseHWConfig {
     return options;
   }
 
+  getToneMapping(videoStream: VideoStreamInfo) {
+    // some drivers (e.g. Mesa radeonsi on AMD) reject yuv420p surfaces and FFmpeg doesn't auto-convert for VAAPI
+    const options = super.getToneMapping(videoStream, 'nv12');
+    if (options.length === 0 && videoStream.pixelFormat !== 'nv12') {
+      options.push('format=nv12');
+    }
+    return options;
+  }
+
   getPresetOptions() {
     let presetIndex = this.getPresetIndex();
     if (presetIndex < 0) {

server/src/utils/misc.spec.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { getKeysDeep, globToPostgresRegex, unsetDeep } from 'src/utils/misc.js';
+import { createLibraryMatcher, getKeysDeep, unsetDeep } from 'src/utils/misc.js';
 
 describe('getKeysDeep', () => {
   it('should handle an empty object', () => {
@@ -52,9 +52,24 @@ describe('unsetDeep', () => {
   });
 });
 
-const matches = (glob: string, value: string) => new RegExp(globToPostgresRegex(glob)).test(value);
+const matches = (glob: string, value: string) =>
+  !createLibraryMatcher({ importPaths: ['/'], exclusionPatterns: [glob] })(value);
+
+describe('createLibraryMatcher', () => {
+  it.each([
+    { path: '/photos/a.jpg', importPaths: ['/photos/'], isInLibrary: true },
+    { path: '/other/a.jpg', importPaths: ['/photos/'], isInLibrary: false },
+    { path: '/other/a.jpg', importPaths: ['/photos/', '/other/'], isInLibrary: true },
+    // unlike SQL `LIKE`, `_` and `%` are literal
+    { path: '/photosX2020/a.jpg', importPaths: ['/photos_2020/'], isInLibrary: false },
+    { path: '/photos/2020/a.jpg', importPaths: ['/photos%/'], isInLibrary: false },
+  ])(
+    'should report $path as in library: $isInLibrary, with import paths $importPaths',
+    ({ path, importPaths, isInLibrary }) => {
+      expect(createLibraryMatcher({ importPaths, exclusionPatterns: [] })(path)).toBe(isInLibrary);
+    },
+  );
 
-describe('globToPostgresRegex', () => {
   const testCases: [string, string, boolean][] = [
     ['**/Raw/**', '/foo/Raw/bar.jpg', true],
     ['**/Raw/**', '/foo/bar.jpg', false],
@@ -69,6 +84,14 @@ describe('globToPostgresRegex', () => {
     // a bare `*` must not cross a path separator, unlike SQL `LIKE`'s `%`
     ['/path/*.*', '/path/photo.jpg', true],
     ['/path/*.*', '/path/2020/photo.jpg', false],
+    // matching is case-insensitive
+    ['**/raw/**', '/foo/RAW/bar.jpg', true],
+    ['**/*.ARW', '/foo/bar.arw', true],
+    // Postgres regexes disagreed with picomatch on these, or rejected them
+    ['**/İstanbul/**', '/foo/istanbul/a.jpg', false],
+    ['**/ΚΎΠΡΟΣ/**', '/foo/Κύπρος/a.jpg', true],
+    [String.raw`**/a\y/**`, '/foo/a/b.jpg', false],
+    [String.raw`**/a\k/**`, '/foo/ak/b.jpg', true],
   ];
 
   it.each(testCases)('should match %s against %s as %s', (glob, value, expected) => {

server/src/utils/misc.ts

@@ -372,10 +372,17 @@ export const useSwagger = (app: INestApplication, { write }: { write: boolean })
   writeFileSync(outputPath, JSON.stringify(patchOpenAPI(openApiDoc), null, 2), { encoding: 'utf8' });
 };
 
-// Compiles a glob to the equivalent Postgres regex (Postgres's Advanced Regular Expression
+// Offline detection and the re-online check must share one matcher, or an asset flips state on every scan
-// dialect is a superset of what picomatch emits, so the two stay in sync with `picomatch.isMatch`,
+export const createLibraryMatcher = ({
-// including which paths a lone `*` may cross vs `/`).
+  importPaths,
-export const globToPostgresRegex = (glob: string) => picomatch.makeRe(glob).source;
+  exclusionPatterns,
+}: {
+  importPaths: string[];
+  exclusionPatterns: string[];
+}) => {
+  const isExcluded = picomatch(exclusionPatterns, { nocase: true });
+  return (path: string) => importPaths.some((importPath) => path.startsWith(importPath)) && !isExcluded(path);
+};
 
 export function clamp(value: number, min: number, max: number) {
   return Math.max(min, Math.min(max, value));

server/test/medium/specs/repositories/asset.repository.spec.ts

@@ -1,10 +1,14 @@
-import { Kysely } from 'kysely';
+import { Insertable, Kysely } from 'kysely';
-import { AssetFileType, AssetOrder, AssetOrderBy, AssetVisibility } from 'src/enum.js';
+import { chunk } from 'lodash-es';
+import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants.js';
+import { AssetFileType, AssetOrder, AssetOrderBy, AssetType, AssetVisibility } from 'src/enum.js';
 import { AssetRepository } from 'src/repositories/asset.repository.js';
+import { LibraryRepository } from 'src/repositories/library.repository.js';
 import { LoggingRepository } from 'src/repositories/logging.repository.js';
 import { DB } from 'src/schema/index.js';
+import { AssetTable } from 'src/schema/tables/asset.table.js';
 import { BaseService } from 'src/services/base.service.js';
-import { newMediumService } from 'test/medium.factory.js';
+import { mediumFactory, newMediumService } from 'test/medium.factory.js';
 import { factory } from 'test/small.factory.js';
 import { getKyselyDB } from 'test/utils.js';
 
@@ -24,6 +28,13 @@ beforeAll(async () => {
   defaultDatabase = await getKyselyDB();
 });
 
+const newExternalLibrary = async (ctx: ReturnType<typeof setup>['ctx']) => {
+  const { user } = await ctx.newUser();
+  return ctx
+    .get(LibraryRepository)
+    .create({ ownerId: user.id, name: 'library', importPaths: [], exclusionPatterns: [] });
+};
+
 // Metadata extraction is repeatable: probing improves, files get repaired,
 // and a re-run has to be able to correct what an earlier run stored.
 const audioRow = (assetId: string, n: number) => ({
@@ -564,10 +575,116 @@ describe(AssetRepository.name, () => {
     });
   });
 
+  describe('findLivePhotoMatch', () => {
+    type LibraryKind = 'upload' | 'external' | 'other external';
+
+    const setupLivePhoto = async ({ still, motion }: { still: LibraryKind; motion: LibraryKind }) => {
+      const { ctx, sut } = setup();
+      const { user } = await ctx.newUser();
+      const newLibrary = () =>
+        ctx
+          .get(LibraryRepository)
+          .create({ ownerId: user.id, name: 'library', importPaths: [], exclusionPatterns: [] });
+      const [external, otherExternal] = await Promise.all([newLibrary(), newLibrary()]);
+      const libraryIds = { upload: null, external: external.id, 'other external': otherExternal.id };
+      const livePhotoCID = 'live-photo-cid';
+
+      const [{ asset: stillAsset }, { asset: motionAsset }] = await Promise.all([
+        ctx.newAsset({ ownerId: user.id, type: AssetType.Image, libraryId: libraryIds[still] }),
+        ctx.newAsset({ ownerId: user.id, type: AssetType.Video, libraryId: libraryIds[motion] }),
+      ]);
+      await Promise.all([
+        ctx.newExif({ assetId: stillAsset.id, livePhotoCID }),
+        ctx.newExif({ assetId: motionAsset.id, livePhotoCID }),
+      ]);
+
+      const match = await sut.findLivePhotoMatch({
+        ownerId: user.id,
+        libraryId: libraryIds[still],
+        livePhotoCID,
+        otherAssetId: stillAsset.id,
+        type: AssetType.Video,
+      });
+
+      return { match, motionAsset };
+    };
+
+    it.each<LibraryKind>(['upload', 'external'])(
+      'should match a motion video in the same %s library',
+      async (library) => {
+        const { match, motionAsset } = await setupLivePhoto({ still: library, motion: library });
+
+        expect(match).toEqual({ id: motionAsset.id, ownerId: motionAsset.ownerId });
+      },
+    );
+
+    it.each<{ still: LibraryKind; motion: LibraryKind }>([
+      { still: 'upload', motion: 'external' },
+      { still: 'external', motion: 'upload' },
+      { still: 'external', motion: 'other external' },
+    ])('should not match a still in the $still library to a motion video in the $motion library', async (libraries) => {
+      const { match } = await setupLivePhoto(libraries);
+
+      expect(match).toBeUndefined();
+    });
+  });
+
   describe('createAll', () => {
     it('should return an empty array when given an empty input', async () => {
       const { sut } = setup();
       await expect(sut.createAll([])).resolves.toStrictEqual([]);
     });
   });
+
+  describe('detectOfflineExternalAssets', () => {
+    it('should offline the assets the matcher rejects and return how many it offlined', async () => {
+      const { ctx, sut } = setup();
+      const [library, otherLibrary] = await Promise.all([newExternalLibrary(ctx), newExternalLibrary(ctx)]);
+      const deletedAt = new Date('2020-01-01T00:00:00.000Z');
+      const newExternalAsset = (originalPath: string, dto: Partial<Insertable<AssetTable>> = {}) =>
+        ctx.newAsset({ ownerId: library.ownerId, libraryId: library.id, isExternal: true, originalPath, ...dto });
+      const [{ asset: kept }, { asset: rejected }, { asset: alreadyOffline }, { asset: otherLibraryAsset }] =
+        await Promise.all([
+          newExternalAsset('/keep/a.jpg'),
+          newExternalAsset('/reject/b.jpg'),
+          newExternalAsset('/reject/c.jpg', { isOffline: true, deletedAt }),
+          ctx.newAsset({
+            ownerId: otherLibrary.ownerId,
+            libraryId: otherLibrary.id,
+            isExternal: true,
+            originalPath: '/reject/d.jpg',
+          }),
+        ]);
+
+      await expect(sut.detectOfflineExternalAssets(library.id, (path) => path.startsWith('/keep/'))).resolves.toBe(1);
+
+      await expect(sut.getById(kept.id)).resolves.toMatchObject({ isOffline: false, deletedAt: null });
+      await expect(sut.getById(rejected.id)).resolves.toMatchObject({ isOffline: true, deletedAt: expect.any(Date) });
+      await expect(sut.getById(alreadyOffline.id)).resolves.toMatchObject({ isOffline: true, deletedAt });
+      await expect(sut.getById(otherLibraryAsset.id)).resolves.toMatchObject({ isOffline: false, deletedAt: null });
+    });
+
+    it('should offline each asset once when the same library is checked concurrently', async () => {
+      const { ctx, sut } = setup();
+      const library = await newExternalLibrary(ctx);
+      const assets = Array.from({ length: JOBS_LIBRARY_PAGINATION_SIZE * 2 }, (_, index) =>
+        mediumFactory.assetInsert({
+          ownerId: library.ownerId,
+          libraryId: library.id,
+          isExternal: true,
+          originalPath: `/reject/${index}.jpg`,
+        }),
+      );
+      for (const batch of chunk(assets, 1000)) {
+        await sut.createAll(batch);
+      }
+
+      const counts = await Promise.all([
+        sut.detectOfflineExternalAssets(library.id, () => false),
+        sut.detectOfflineExternalAssets(library.id, () => false),
+      ]);
+
+      expect(counts[0] + counts[1]).toBe(assets.length);
+    }, 60_000);
+  });
 });

server/test/medium/specs/services/library.service.spec.ts

@@ -1,9 +1,11 @@
 import { Kysely } from 'kysely';
+import { chunk } from 'lodash-es';
 import { existsSync } from 'node:fs';
 import { copyFile, mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { dirname, join, resolve } from 'node:path';
 import type { SystemConfig } from 'src/dtos/config.dto.js';
+import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants.js';
 import { StorageCore } from 'src/cores/storage.core.js';
 import { AssetStatus, JobName, JobStatus } from 'src/enum.js';
 import { AssetJobRepository } from 'src/repositories/asset-job.repository.js';
@@ -20,7 +22,7 @@ import { WebsocketRepository } from 'src/repositories/websocket.repository.js';
 import { DB } from 'src/schema/index.js';
 import { LibraryService } from 'src/services/library.service.js';
 import { systemConfigStub } from 'test/fixtures/system-config.stub.js';
-import { MediumTestContext, testAssetsDir } from 'test/medium.factory.js';
+import { MediumTestContext, mediumFactory, testAssetsDir } from 'test/medium.factory.js';
 import { newUuid } from 'test/small.factory.js';
 import { getKyselyDB } from 'test/utils.js';
 
@@ -574,6 +576,44 @@ describe(LibraryService.name, () => {
       await ctx.scan(library.id);
       await expect(ctx.getAssetPaths(library.id)).resolves.toEqual([asset1]);
     });
+
+    // Postgres case folding disagreed with picomatch on these: `İ` folds to `i`, but `Σ` does not fold to `ς`
+    it.each([
+      { pattern: '**/İstanbul/**', folder: 'istanbul', isExcluded: false },
+      { pattern: '**/ΚΎΠΡΟΣ/**', folder: 'Κύπρος', isExcluded: true },
+    ])('should apply $pattern to $folder the same way on every scan', async ({ pattern, folder, isExcluded }) => {
+      const { sut, ctx } = setup();
+
+      const asset = await createFile(join(importRoot, folder, 'asset.png'));
+      const library = await ctx.createLibrary({ importPaths: [importRoot] });
+
+      await ctx.scan(library.id);
+      await sut.update(library.id, { exclusionPatterns: [pattern] });
+
+      const expected = isExcluded ? [] : [asset];
+      await ctx.scan(library.id);
+      await expect(ctx.getAssetPaths(library.id)).resolves.toEqual(expected);
+      await ctx.scan(library.id);
+      await expect(ctx.getAssetPaths(library.id)).resolves.toEqual(expected);
+    });
+
+    it('should offline a missing asset when an exclusion pattern is not a valid Postgres regex', async () => {
+      const { ctx } = setup();
+
+      const kept = await createFile(join(importPath, 'kept.png'));
+      const missing = await createFile(join(importPath, 'missing.png'));
+      const library = await ctx.createLibrary({
+        importPaths: [importPath],
+        exclusionPatterns: [String.raw`**/a\k/**`],
+      });
+
+      await ctx.scan(library.id);
+      await expect(ctx.getAssetPaths(library.id)).resolves.toEqual([kept, missing]);
+
+      await rm(missing);
+      await ctx.scan(library.id);
+      await expect(ctx.getAssetPaths(library.id)).resolves.toEqual([kept]);
+    });
   });
 
   describe('watch', () => {
@@ -646,6 +686,35 @@ describe(LibraryService.name, () => {
       expect(updated).toEqual(expect.objectContaining({ isOffline: true }));
       expect(updated?.deletedAt).toBeInstanceOf(Date);
     });
+
+    it('should not exhaust the database pool when more libraries are checked at once than it has connections', async () => {
+      const { sut, ctx } = setup();
+      const assetRepo = ctx.get(AssetRepository);
+      // the pool has 10 connections, and each library spans more than one batch so it offlines assets mid-read
+      const libraries = await Promise.all(
+        Array.from({ length: 10 }, () => ctx.createLibrary({ importPaths: [importPath] })),
+      );
+      for (const library of libraries) {
+        const assets = Array.from({ length: JOBS_LIBRARY_PAGINATION_SIZE + 1 }, (_, index) =>
+          mediumFactory.assetInsert({
+            ownerId: library.ownerId,
+            libraryId: library.id,
+            originalPath: join(outsidePath, `${index}.jpg`),
+            isExternal: true,
+          }),
+        );
+        for (const batch of chunk(assets, 1000)) {
+          await assetRepo.createAll(batch);
+        }
+      }
+
+      await expect(Promise.all(libraries.map(({ id }) => sut.handleQueueSyncAssets({ id })))).resolves.toEqual(
+        libraries.map(() => JobStatus.Success),
+      );
+      for (const { id } of libraries) {
+        await expect(ctx.getAssetPaths(id)).resolves.toEqual([]);
+      }
+    }, 60_000);
   });
 
   describe('handleSyncAssets', () => {

web/package.json

@@ -33,7 +33,7 @@
   },
   "dependencies": {
     "@formatjs/icu-messageformat-parser": "^3.0.0",
-    "@futo-org/backups-orchestrator-ui": "1.11.5",
+    "@futo-org/backups-orchestrator-ui": "1.12.1",
     "@immich/justified-layout-wasm": "^0.4.3",
     "@immich/sdk": "workspace:*",
     "@immich/ui": "^0.91.0",