MertandGitHub
23f1f87dd0
* add immich_model * graph per shape * decouple ocr from ort * prepared graphs * use f16 for accelerators * preload all shapes * warm-up pass after preload * keep precision for fp32 graphs * ensure padding for text recognition * optimize postprocessing for fp16 outputs * handle raw ctc logits * use f16 output for rknn * mmap token embedding * read device metadata
88 lines
3.3 KiB
Python
88 lines
3.3 KiB
Python
import json
|
|
from abc import abstractmethod
|
|
from functools import cached_property
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
from numpy.typing import NDArray
|
|
from PIL import Image
|
|
|
|
from immich_ml.config import log
|
|
from immich_ml.models.base import InferenceModel
|
|
from immich_ml.models.transforms import (
|
|
crop_pil,
|
|
decode_pil,
|
|
get_pil_resampling,
|
|
normalize,
|
|
resize_pil,
|
|
serialize_np_array,
|
|
to_numpy,
|
|
)
|
|
from immich_ml.schemas import ModelGraph, ModelInput, ModelSession, ModelTask, ModelType
|
|
|
|
|
|
class BaseCLIPVisualEncoder(InferenceModel):
|
|
depends = []
|
|
identity = (ModelType.VISUAL, ModelTask.SEARCH)
|
|
|
|
def _predict(self, inputs: Image.Image | bytes) -> str:
|
|
image = decode_pil(inputs)
|
|
session = self.session.for_shape(self.shape_policy.dims[0])
|
|
res: NDArray[np.float32] = session.run(None, self.transform(session, image))[0][0]
|
|
return serialize_np_array(res)
|
|
|
|
@abstractmethod
|
|
def transform(self, session: ModelGraph, image: Image.Image) -> ModelInput:
|
|
pass
|
|
|
|
@property
|
|
def model_cfg_path(self) -> Path:
|
|
return self.cache_dir / "config.json"
|
|
|
|
@property
|
|
def preprocess_cfg_path(self) -> Path:
|
|
return self.model_dir / "preprocess_cfg.json"
|
|
|
|
@cached_property
|
|
def model_cfg(self) -> dict[str, Any]:
|
|
log.debug(f"Loading model config for CLIP model '{self.model_name}'")
|
|
model_cfg: dict[str, Any] = json.load(self.model_cfg_path.open(encoding="utf-8"))
|
|
log.debug(f"Loaded model config for CLIP model '{self.model_name}'")
|
|
return model_cfg
|
|
|
|
@cached_property
|
|
def preprocess_cfg(self) -> dict[str, Any]:
|
|
log.debug(f"Loading visual preprocessing config for CLIP model '{self.model_name}'")
|
|
preprocess_cfg: dict[str, Any] = json.load(self.preprocess_cfg_path.open(encoding="utf-8"))
|
|
log.debug(f"Loaded visual preprocessing config for CLIP model '{self.model_name}'")
|
|
return preprocess_cfg
|
|
|
|
|
|
class OpenClipVisualEncoder(BaseCLIPVisualEncoder):
|
|
def _load(self) -> ModelSession:
|
|
size: list[int] | int = self.preprocess_cfg["size"]
|
|
self.size = size[0] if isinstance(size, list) else size
|
|
|
|
self.resampling = get_pil_resampling(self.preprocess_cfg["interpolation"])
|
|
self.mean = np.array(self.preprocess_cfg["mean"], dtype=np.float32)
|
|
self.std = np.array(self.preprocess_cfg["std"], dtype=np.float32)
|
|
|
|
return super()._load()
|
|
|
|
def transform(self, session: ModelGraph, image: Image.Image) -> ModelInput:
|
|
image = self._resize(image)
|
|
if session.normalizes_input:
|
|
rgb: NDArray[np.uint8] = np.asarray(image if image.mode == "RGB" else image.convert("RGB"))
|
|
return {"image": rgb[None]}
|
|
return {"image": normalize(to_numpy(image), self.mean, self.std).transpose(2, 0, 1)[None]}
|
|
|
|
def _resize(self, image: Image.Image) -> Image.Image:
|
|
match self.preprocess_cfg.get("resize_mode", "shortest"): # open_clip's default for older configs
|
|
case "squash":
|
|
return image.resize((self.size, self.size), resample=self.resampling)
|
|
case "shortest":
|
|
return crop_pil(resize_pil(image, self.size, self.resampling), self.size)
|
|
case mode:
|
|
raise ValueError(f"Unsupported resize_mode {mode!r} in {self.preprocess_cfg_path}")
|