diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index b790705cc..dbdae444d 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -531,7 +531,7 @@ where and "libraryId" = $5::uuid and ( not "originalPath" like $6 - or "originalPath" like $7 + or "originalPath" ~ $7 ) -- AssetRepository.filterNewExternalAssetPaths diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 9427bfca6..189c5c621 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -54,7 +54,7 @@ import { withTagId, withTags, } from 'src/utils/database'; -import { globToSqlPattern } from 'src/utils/misc'; +import { globToPostgresRegex } from 'src/utils/misc'; export type AssetStats = Record; @@ -1086,7 +1086,7 @@ export class AssetRepository { exclusionPatterns: string[], ): Promise { const paths = importPaths.map((importPath) => `${importPath}%`); - const exclusions = exclusionPatterns.map((pattern) => globToSqlPattern(pattern)); + const exclusions = exclusionPatterns.map((pattern) => globToPostgresRegex(pattern)); return this.db .updateTable('asset') @@ -1100,7 +1100,7 @@ export class AssetRepository { .where((eb) => eb.or([ eb.not(eb.or(paths.map((path) => eb('originalPath', 'like', path)))), - eb.or(exclusions.map((path) => eb('originalPath', 'like', path))), + eb.or(exclusions.map((pattern) => eb('originalPath', '~', pattern))), ]), ) .executeTakeFirstOrThrow(); diff --git a/server/src/utils/misc.spec.ts b/server/src/utils/misc.spec.ts index 3c4548293..e6b36d4a0 100644 --- a/server/src/utils/misc.spec.ts +++ b/server/src/utils/misc.spec.ts @@ -1,4 +1,4 @@ -import { getKeysDeep, globToSqlPattern, unsetDeep } from 'src/utils/misc'; +import { getKeysDeep, globToPostgresRegex, unsetDeep } from 'src/utils/misc'; import { describe, expect, it } from 'vitest'; describe('getKeysDeep', () => { @@ -52,18 +52,26 @@ describe('unsetDeep', () => { }); }); -describe('globToSqlPattern', () => { - const testCases = [ - ['**/Raw/**', '%/Raw/%'], - ['**/abc/*.tif', '%/abc/%.tif'], - ['**/*.tif', '%/%.tif'], - ['**/*.jp?', '%/%.jp_'], - ['**/@eaDir/**', '%/@eaDir/%'], - ['**/._*', `%/._%`], - ['/absolute/path/**', `/absolute/path/%`], +const matches = (glob: string, value: string) => new RegExp(globToPostgresRegex(glob)).test(value); + +describe('globToPostgresRegex', () => { + const testCases: [string, string, boolean][] = [ + ['**/Raw/**', '/foo/Raw/bar.jpg', true], + ['**/Raw/**', '/foo/bar.jpg', false], + ['**/abc/*.tif', '/foo/abc/scan.tif', true], + ['**/abc/*.tif', '/foo/abc/sub/scan.tif', false], + ['**/*.tif', '/foo/bar.tif', true], + ['**/*.jp?', '/foo/bar.jpg', true], + ['**/@eaDir/**', '/foo/@eaDir/thumb.jpg', true], + ['**/._*', '/foo/._resource', true], + ['/absolute/path/**', '/absolute/path/photo.jpg', true], + ['/absolute/path/**', '/other/path/photo.jpg', false], + // a bare `*` must not cross a path separator, unlike SQL `LIKE`'s `%` + ['/path/*.*', '/path/photo.jpg', true], + ['/path/*.*', '/path/2020/photo.jpg', false], ]; - it.each(testCases)('should convert %s to %s', (input, expected) => { - expect(globToSqlPattern(input)).toEqual(expected); + it.each(testCases)('should match %s against %s as %s', (glob, value, expected) => { + expect(matches(glob, value)).toEqual(expected); }); }); diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 84ebc7e98..7727a8eb8 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -12,7 +12,6 @@ import { cleanupOpenApiDoc } from 'nestjs-zod'; import { writeFileSync } from 'node:fs'; import path from 'node:path'; import picomatch from 'picomatch'; -import parse from 'picomatch/lib/parse'; import { CLIP_MODEL_INFO, JOBS_ASSET_PAGINATION_SIZE, endpointTags, serverVersion } from 'src/constants'; import { extraModels } from 'src/decorators'; import { SystemConfig } from 'src/dtos/config.dto'; @@ -338,37 +337,10 @@ export const useSwagger = (app: INestApplication, { write }: { write: boolean }) } }; -const convertTokenToSqlPattern = (token: parse.Token): string => { - switch (token.type) { - case 'slash': { - return '/'; - } - case 'text': { - return token.value; - } - case 'globstar': - case 'star': { - return '%'; - } - case 'underscore': { - return String.raw`\_`; - } - case 'qmark': { - return '_'; - } - case 'dot': { - return '.'; - } - default: { - return ''; - } - } -}; - -export const globToSqlPattern = (glob: string) => { - const tokens = picomatch.parse(glob).tokens; - return tokens.map((token) => convertTokenToSqlPattern(token)).join(''); -}; +// Compiles a glob to the equivalent Postgres regex (Postgres's Advanced Regular Expression +// dialect is a superset of what picomatch emits, so the two stay in sync with `picomatch.isMatch`, +// including which paths a lone `*` may cross vs `/`). +export const globToPostgresRegex = (glob: string) => picomatch.makeRe(glob).source; export function clamp(value: number, min: number, max: number) { return Math.max(min, Math.min(max, value));