mirror of
https://github.com/immich-app/immich.git
synced 2025-02-03 01:22:44 +01:00
Compare commits
85 commits
06140ba078
...
209a67c312
Author | SHA1 | Date | |
---|---|---|---|
|
209a67c312 | ||
|
c2ba1cc202 | ||
|
08db77db23 | ||
|
6e08f1f371 | ||
|
3ed5d3dbac | ||
|
f9387d8478 | ||
|
4bb98471f4 | ||
|
794da29411 | ||
|
dd52c2d68c | ||
|
59e4b6598e | ||
|
20ba9f97e9 | ||
|
ac4ce3ea9c | ||
|
2b967ca358 | ||
|
1653cd9cd7 | ||
|
32f3707e52 | ||
|
58f1cc92d7 | ||
|
d2b7e10f55 | ||
|
b3ae5d34cc | ||
|
4e42fbc091 | ||
|
9926045d5e | ||
|
d7381ab5c1 | ||
|
87a46dcc5e | ||
|
be76857ae6 | ||
|
f5de3de163 | ||
|
3634ae1f5b | ||
|
05675921be | ||
|
f067212491 | ||
|
bc48b67379 | ||
|
26d5fb0ac6 | ||
|
f32d991131 | ||
|
9882b83cd4 | ||
|
01eb09526e | ||
|
b5a4ed5160 | ||
|
0f03f77e8e | ||
|
c21ce40d9c | ||
|
cb01a11f19 | ||
|
5244ed6d4d | ||
|
8b80d034cb | ||
|
4b0f93cf6a | ||
|
6c4e6cb96f | ||
|
b6cc2054c5 | ||
|
f328104e84 | ||
|
2f7e44aa63 | ||
|
ebdfe1b7b6 | ||
|
daf886088a | ||
|
4c7ac1438b | ||
|
8965a9fb16 | ||
|
7ae4b7129d | ||
|
1775397a84 | ||
|
68fccad462 | ||
|
bb67a9db6e | ||
|
c109e28686 | ||
|
e6ff21b345 | ||
|
c665fd2625 | ||
|
c72cf61ed0 | ||
|
19ee48f6f0 | ||
|
efaf70eb9d | ||
|
665718b09e | ||
|
807111e3b5 | ||
|
815ed1ae66 | ||
|
416211916d | ||
|
23d0ea0e7b | ||
|
d5e453a773 | ||
|
7f2af6f819 | ||
|
f4671f4886 | ||
|
7aaf3aa57b | ||
|
506ca0d3a4 | ||
|
d5ef821b24 | ||
|
d10147f478 | ||
|
66004e3b83 | ||
|
c20d110257 | ||
|
a2722e16e7 | ||
|
4d704e9f73 | ||
|
9bc3e5b2e2 | ||
|
8608b9c6c5 | ||
|
a94fad543b | ||
|
082c426e34 | ||
|
da152bd284 | ||
|
257cc6c963 | ||
|
4140e93aea | ||
|
b6c4b37237 | ||
|
bc849e2e9f | ||
|
7fddf282cf | ||
|
6ffc227330 | ||
|
8ef3e49f74 |
28 changed files with 513 additions and 52 deletions
.github/workflows
docker
docs
docs
src/components
machine-learning
web
7
.github/workflows/docker.yml
vendored
7
.github/workflows/docker.yml
vendored
|
@ -48,7 +48,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
suffix: ["", "-cuda", "-openvino", "-armnn"]
|
||||
suffix: ["", "-cuda", "-openvino", "-armnn","-rknn"]
|
||||
steps:
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
|
@ -116,6 +116,9 @@ jobs:
|
|||
- platforms: linux/arm64
|
||||
device: armnn
|
||||
suffix: -armnn
|
||||
- platforms: linux/arm64
|
||||
device: rknn
|
||||
suffix: -rknn
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
@ -307,4 +310,4 @@ jobs:
|
|||
run: exit 1
|
||||
- name: All jobs passed or skipped
|
||||
if: ${{ !(contains(needs.*.result, 'failure')) }}
|
||||
run: echo "All jobs passed or skipped" && echo "${{ toJSON(needs.*.result) }}"
|
||||
run: echo "All jobs passed or skipped" && echo "${{ toJSON(needs.*.result) }}"
|
2
.github/workflows/test.yml
vendored
2
.github/workflows/test.yml
vendored
|
@ -390,7 +390,7 @@ jobs:
|
|||
poetry run black --check app export
|
||||
- name: Run mypy type checking
|
||||
run: |
|
||||
poetry run mypy --install-types --non-interactive --strict app/
|
||||
mkdir .mypy_cache && poetry run mypy --install-types --non-interactive --strict app/
|
||||
- name: Run tests and coverage
|
||||
run: |
|
||||
poetry run pytest app --cov=app --cov-report term-missing
|
||||
|
|
|
@ -86,12 +86,12 @@ services:
|
|||
image: immich-machine-learning-dev:latest
|
||||
# extends:
|
||||
# file: hwaccel.ml.yml
|
||||
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
|
||||
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference
|
||||
build:
|
||||
context: ../machine-learning
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
- DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
|
||||
- DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference
|
||||
ports:
|
||||
- 3003:3003
|
||||
volumes:
|
||||
|
|
|
@ -29,12 +29,12 @@ services:
|
|||
image: immich-machine-learning:latest
|
||||
# extends:
|
||||
# file: hwaccel.ml.yml
|
||||
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
|
||||
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference
|
||||
build:
|
||||
context: ../machine-learning
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
- DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
|
||||
- DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference
|
||||
ports:
|
||||
- 3003:3003
|
||||
volumes:
|
||||
|
@ -68,22 +68,12 @@ services:
|
|||
- 5432:5432
|
||||
healthcheck:
|
||||
test: >-
|
||||
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
|
||||
Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
|
||||
--command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
|
||||
echo "checksum failure count is $$Chksum";
|
||||
[ "$$Chksum" = '0' ] || exit 1
|
||||
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1; Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1
|
||||
interval: 5m
|
||||
start_interval: 30s
|
||||
start_period: 5m
|
||||
command: >-
|
||||
postgres
|
||||
-c shared_preload_libraries=vectors.so
|
||||
-c 'search_path="$$user", public, vectors'
|
||||
-c logging_collector=on
|
||||
-c max_wal_size=2GB
|
||||
-c shared_buffers=512MB
|
||||
-c wal_compression=on
|
||||
postgres -c shared_preload_libraries=vectors.so -c 'search_path="$$user", public, vectors' -c logging_collector=on -c max_wal_size=2GB -c shared_buffers=512MB -c wal_compression=on
|
||||
restart: always
|
||||
|
||||
# set IMMICH_TELEMETRY_INCLUDE=all in .env to enable metrics
|
||||
|
@ -100,7 +90,7 @@ services:
|
|||
# add data source for http://immich-prometheus:9090 to get started
|
||||
immich-grafana:
|
||||
container_name: immich_grafana
|
||||
command: ['./run.sh', '-disable-reporting']
|
||||
command: [ './run.sh', '-disable-reporting' ]
|
||||
ports:
|
||||
- 3000:3000
|
||||
image: grafana/grafana:11.4.0-ubuntu@sha256:afccec22ba0e4815cca1d2bf3836e414322390dc78d77f1851976ffa8d61051c
|
||||
|
|
|
@ -32,12 +32,12 @@ services:
|
|||
|
||||
immich-machine-learning:
|
||||
container_name: immich_machine_learning
|
||||
# For hardware acceleration, add one of -[armnn, cuda, openvino] to the image tag.
|
||||
# For hardware acceleration, add one of -[armnn, cuda, openvino, rknn] to the image tag.
|
||||
# Example tag: ${IMMICH_VERSION:-release}-cuda
|
||||
image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release}
|
||||
# extends: # uncomment this section for hardware acceleration - see https://immich.app/docs/features/ml-hardware-acceleration
|
||||
# file: hwaccel.ml.yml
|
||||
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference - use the `-wsl` version for WSL2 where applicable
|
||||
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference - use the `-wsl` version for WSL2 where applicable
|
||||
volumes:
|
||||
- model-cache:/cache
|
||||
env_file:
|
||||
|
@ -66,22 +66,12 @@ services:
|
|||
- ${DB_DATA_LOCATION}:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: >-
|
||||
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
|
||||
Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
|
||||
--command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
|
||||
echo "checksum failure count is $$Chksum";
|
||||
[ "$$Chksum" = '0' ] || exit 1
|
||||
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1; Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1
|
||||
interval: 5m
|
||||
start_interval: 30s
|
||||
start_period: 5m
|
||||
command: >-
|
||||
postgres
|
||||
-c shared_preload_libraries=vectors.so
|
||||
-c 'search_path="$$user", public, vectors'
|
||||
-c logging_collector=on
|
||||
-c max_wal_size=2GB
|
||||
-c shared_buffers=512MB
|
||||
-c wal_compression=on
|
||||
postgres -c shared_preload_libraries=vectors.so -c 'search_path="$$user", public, vectors' -c logging_collector=on -c max_wal_size=2GB -c shared_buffers=512MB -c wal_compression=on
|
||||
restart: always
|
||||
|
||||
volumes:
|
||||
|
|
|
@ -13,6 +13,14 @@ services:
|
|||
volumes:
|
||||
- /lib/firmware/mali_csffw.bin:/lib/firmware/mali_csffw.bin:ro # Mali firmware for your chipset (not always required depending on the driver)
|
||||
- /usr/lib/libmali.so:/usr/lib/libmali.so:ro # Mali driver for your chipset (always required)
|
||||
|
||||
rknn:
|
||||
security_opt:
|
||||
- systempaths=unconfined
|
||||
devices:
|
||||
- /dev/dri:/dev/dri
|
||||
volumes:
|
||||
- /sys/kernel/debug/:/sys/kernel/debug/:ro
|
||||
|
||||
cpu: {}
|
||||
|
||||
|
|
|
@ -12,6 +12,7 @@ You do not need to redo any machine learning jobs after enabling hardware accele
|
|||
- ARM NN (Mali)
|
||||
- CUDA (NVIDIA GPUs with [compute capability](https://developer.nvidia.com/cuda-gpus) 5.2 or higher)
|
||||
- OpenVINO (Intel discrete GPUs such as Iris Xe and Arc)
|
||||
- RKNN (Rockchip)
|
||||
|
||||
## Limitations
|
||||
|
||||
|
@ -46,6 +47,15 @@ You do not need to redo any machine learning jobs after enabling hardware accele
|
|||
- The server must have a discrete GPU, i.e. Iris Xe or Arc. Expect issues when attempting to use integrated graphics.
|
||||
- Ensure the server's kernel version is new enough to use the device for hardware accceleration.
|
||||
|
||||
#### RKNN
|
||||
|
||||
- You must have a supported Rockchip SoC, only RK3566 and RK3588 are supported at this moment.
|
||||
- Make sure you have the appropriate linux kernel driver installed
|
||||
- This is usually pre-installed on the device vendor's Linux images
|
||||
- RKNPU driver V0.9.8 or later must be available in the host server
|
||||
- You may confirm this by running `cat /sys/kernel/debug/rknpu/version` to check the version
|
||||
- Optional: Configure your `.env` file, see [environment variables](/docs/install/environment-variables) for RKNN specific settings
|
||||
|
||||
## Setup
|
||||
|
||||
1. If you do not already have it, download the latest [`hwaccel.ml.yml`][hw-file] file and ensure it's in the same folder as the `docker-compose.yml`.
|
||||
|
|
|
@ -168,6 +168,8 @@ Redis (Sentinel) URL example JSON before encoding:
|
|||
| `MACHINE_LEARNING_ANN_TUNING_LEVEL` | ARM-NN GPU tuning level (1: rapid, 2: normal, 3: exhaustive) | `2` | machine learning |
|
||||
| `MACHINE_LEARNING_DEVICE_IDS`<sup>\*4</sup> | Device IDs to use in multi-GPU environments | `0` | machine learning |
|
||||
| `MACHINE_LEARNING_MAX_BATCH_SIZE__FACIAL_RECOGNITION` | Set the maximum number of faces that will be processed at once by the facial recognition model | None (`1` if using OpenVINO) | machine learning |
|
||||
| `MACHINE_LEARNING_RKNN` | Enable RKNN hardware acceleration if supported | `True` | machine learning |
|
||||
| `MACHINE_LEARNING_RKNN_THREADS` | How many threads of RKNN runtime should be spinned up while inferencing. | `1` | machine learning |
|
||||
|
||||
\*1: It is recommended to begin with this parameter when changing the concurrency levels of the machine learning service and then tune the other ones.
|
||||
|
||||
|
|
|
@ -99,6 +99,11 @@ const projects: CommunityProjectProps[] = [
|
|||
description: 'Downloads a configurable number of random photos based on people or album ID.',
|
||||
url: 'https://github.com/jon6fingrs/immich-dl',
|
||||
},
|
||||
{
|
||||
title: 'Immich Upload Optimizer',
|
||||
description: 'Automatically optimize files uploaded to Immich in order to save storage space',
|
||||
url: 'https://github.com/miguelangel-nubla/immich-upload-optimizer',
|
||||
},
|
||||
];
|
||||
|
||||
function CommunityProject({ title, description, url }: CommunityProjectProps): JSX.Element {
|
||||
|
|
|
@ -15,6 +15,8 @@ RUN mkdir /opt/armnn && \
|
|||
cd /opt/ann && \
|
||||
sh build.sh
|
||||
|
||||
FROM builder-cpu AS builder-rknn
|
||||
|
||||
FROM builder-${DEVICE} AS builder
|
||||
|
||||
ARG DEVICE
|
||||
|
@ -80,6 +82,10 @@ COPY --from=builder-armnn \
|
|||
/opt/ann/build.sh \
|
||||
/opt/armnn/
|
||||
|
||||
FROM prod-cpu AS prod-rknn
|
||||
|
||||
ADD --checksum=sha256:73993ed4b440460825f21611731564503cc1d5a0c123746477da6cd574f34885 https://github.com/airockchip/rknn-toolkit2/raw/refs/tags/v2.3.0/rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so /usr/lib/
|
||||
|
||||
FROM prod-${DEVICE} AS prod
|
||||
ARG DEVICE
|
||||
|
||||
|
@ -104,9 +110,10 @@ RUN echo "hard core 0" >> /etc/security/limits.conf && \
|
|||
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
COPY ann/ann.py /usr/src/ann/ann.py
|
||||
COPY rknn/rknnpool.py /usr/src/rknn/rknnpool.py
|
||||
COPY start.sh log_conf.json gunicorn_conf.py ./
|
||||
COPY app .
|
||||
ENTRYPOINT ["tini", "--"]
|
||||
CMD ["./start.sh"]
|
||||
|
||||
HEALTHCHECK CMD python3 healthcheck.py
|
||||
HEALTHCHECK CMD python3 healthcheck.py
|
|
@ -64,6 +64,8 @@ class Settings(BaseSettings):
|
|||
ann: bool = True
|
||||
ann_fp16_turbo: bool = False
|
||||
ann_tuning_level: int = 2
|
||||
rknn: bool = True
|
||||
rknn_threads: int = 1
|
||||
preload: PreloadModelData | None = None
|
||||
max_batch_size: MaxBatchSize | None = None
|
||||
|
||||
|
|
|
@ -136,6 +136,12 @@ def ann_session() -> Iterator[mock.Mock]:
|
|||
yield mocked
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def rknn_session() -> Iterator[mock.Mock]:
|
||||
with mock.patch("app.sessions.rknn.RknnPoolExecutor") as mocked:
|
||||
yield mocked
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def rmtree() -> Iterator[mock.Mock]:
|
||||
with mock.patch("app.models.base.rmtree", autospec=True) as mocked:
|
||||
|
|
|
@ -8,7 +8,9 @@ from typing import Any, ClassVar
|
|||
from huggingface_hub import snapshot_download
|
||||
|
||||
import ann.ann
|
||||
import rknn.rknnpool
|
||||
from app.sessions.ort import OrtSession
|
||||
from app.sessions.rknn import RknnSession
|
||||
|
||||
from ..config import clean_name, log, settings
|
||||
from ..schemas import ModelFormat, ModelIdentity, ModelSession, ModelTask, ModelType
|
||||
|
@ -66,12 +68,17 @@ class InferenceModel(ABC):
|
|||
pass
|
||||
|
||||
def _download(self) -> None:
|
||||
ignore_patterns = [] if self.model_format == ModelFormat.ARMNN else ["*.armnn"]
|
||||
ignored_patterns: dict[ModelFormat, list[str]] = {
|
||||
ModelFormat.ONNX: ["*.armnn", "*.rknn"],
|
||||
ModelFormat.ARMNN: ["*.rknn"],
|
||||
ModelFormat.RKNN: ["*.armnn"],
|
||||
}
|
||||
|
||||
snapshot_download(
|
||||
f"immich-app/{clean_name(self.model_name)}",
|
||||
cache_dir=self.cache_dir,
|
||||
local_dir=self.cache_dir,
|
||||
ignore_patterns=ignore_patterns,
|
||||
ignore_patterns=ignored_patterns.get(self.model_format, []),
|
||||
)
|
||||
|
||||
def _load(self) -> ModelSession:
|
||||
|
@ -108,6 +115,8 @@ class InferenceModel(ABC):
|
|||
session: ModelSession = AnnSession(model_path)
|
||||
case ".onnx":
|
||||
session = OrtSession(model_path)
|
||||
case ".rknn":
|
||||
session = RknnSession(model_path)
|
||||
case _:
|
||||
raise ValueError(f"Unsupported model file type: {model_path.suffix}")
|
||||
return session
|
||||
|
@ -155,4 +164,9 @@ class InferenceModel(ABC):
|
|||
|
||||
@property
|
||||
def _model_format_default(self) -> ModelFormat:
|
||||
return ModelFormat.ARMNN if ann.ann.is_available and settings.ann else ModelFormat.ONNX
|
||||
if rknn.rknnpool.is_available and settings.rknn:
|
||||
return ModelFormat.RKNN
|
||||
elif ann.ann.is_available and settings.ann:
|
||||
return ModelFormat.ARMNN
|
||||
else:
|
||||
return ModelFormat.ONNX
|
||||
|
|
|
@ -35,6 +35,7 @@ class ModelType(StrEnum):
|
|||
class ModelFormat(StrEnum):
|
||||
ARMNN = "armnn"
|
||||
ONNX = "onnx"
|
||||
RKNN = "rknn"
|
||||
|
||||
|
||||
class ModelSource(StrEnum):
|
||||
|
|
73
machine-learning/app/sessions/rknn.py
Normal file
73
machine-learning/app/sessions/rknn.py
Normal file
|
@ -0,0 +1,73 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
from numpy.typing import NDArray
|
||||
|
||||
from app.schemas import SessionNode
|
||||
from rknn.rknnpool import RknnPoolExecutor, soc_name
|
||||
|
||||
from ..config import log, settings
|
||||
|
||||
|
||||
def runInference(rknn_lite: Any, input: list[NDArray[np.float32]]) -> list[NDArray[np.float32]]:
|
||||
outputs: list[NDArray[np.float32]] = rknn_lite.inference(inputs=input, data_format="nchw")
|
||||
|
||||
return outputs
|
||||
|
||||
|
||||
input_output_mapping: dict[str, dict[str, Any]] = {
|
||||
"detection": {
|
||||
"input": {"norm_tensor:0": (1, 3, 640, 640)},
|
||||
"output": {
|
||||
"norm_tensor:1": (12800, 1),
|
||||
"norm_tensor:2": (3200, 1),
|
||||
"norm_tensor:3": (800, 1),
|
||||
"norm_tensor:4": (12800, 4),
|
||||
"norm_tensor:5": (3200, 4),
|
||||
"norm_tensor:6": (800, 4),
|
||||
"norm_tensor:7": (12800, 10),
|
||||
"norm_tensor:8": (3200, 10),
|
||||
"norm_tensor:9": (800, 10),
|
||||
},
|
||||
},
|
||||
"recognition": {"input": {"norm_tensor:0": (1, 3, 112, 112)}, "output": {"norm_tensor:1": (1, 512)}},
|
||||
}
|
||||
|
||||
|
||||
class RknnSession:
|
||||
def __init__(self, model_path: Path | str):
|
||||
self.model_path = Path(str(model_path).replace("model", soc_name))
|
||||
self.model_type = "detection" if "detection" in self.model_path.as_posix() else "recognition"
|
||||
self.tpe = settings.rknn_threads
|
||||
|
||||
log.info(f"Loading RKNN model from {self.model_path} with {self.tpe} threads.")
|
||||
self.rknnpool = RknnPoolExecutor(rknnModel=self.model_path.as_posix(), tpes=self.tpe, func=runInference)
|
||||
log.info(f"Loaded RKNN model from {self.model_path} with {self.tpe} threads.")
|
||||
|
||||
def __del__(self) -> None:
|
||||
self.rknnpool.release()
|
||||
|
||||
def get_inputs(self) -> list[SessionNode]:
|
||||
return [RknnNode(name=k, shape=v) for k, v in input_output_mapping[self.model_type]["input"].items()]
|
||||
|
||||
def get_outputs(self) -> list[SessionNode]:
|
||||
return [RknnNode(name=k, shape=v) for k, v in input_output_mapping[self.model_type]["output"].items()]
|
||||
|
||||
def run(
|
||||
self,
|
||||
output_names: list[str] | None,
|
||||
input_feed: dict[str, NDArray[np.float32]] | dict[str, NDArray[np.int32]],
|
||||
run_options: Any = None,
|
||||
) -> list[NDArray[np.float32]]:
|
||||
input_data: list[NDArray[np.float32]] = [np.ascontiguousarray(v) for v in input_feed.values()]
|
||||
self.rknnpool.put(input_data)
|
||||
outputs: list[NDArray[np.float32]] = self.rknnpool.get()
|
||||
return outputs
|
||||
|
||||
|
||||
class RknnNode(NamedTuple):
|
||||
name: str | None
|
||||
shape: tuple[int, ...]
|
|
@ -25,6 +25,7 @@ from app.models.facial_recognition.detection import FaceDetector
|
|||
from app.models.facial_recognition.recognition import FaceRecognizer
|
||||
from app.sessions.ann import AnnSession
|
||||
from app.sessions.ort import OrtSession
|
||||
from app.sessions.rknn import RknnSession, runInference
|
||||
|
||||
from .config import Settings, settings
|
||||
from .models.base import InferenceModel
|
||||
|
@ -69,6 +70,14 @@ class TestBase:
|
|||
|
||||
assert encoder.model_format == ModelFormat.ARMNN
|
||||
|
||||
def test_sets_default_model_format_rknn(self, mocker: MockerFixture) -> None:
|
||||
mocker.patch.object(settings, "rknn", True)
|
||||
mocker.patch("rknn.rknnpool.is_available", False)
|
||||
|
||||
encoder = OpenClipTextualEncoder("ViT-B-32__openai")
|
||||
|
||||
assert encoder.model_format == ModelFormat.ONNX
|
||||
|
||||
def test_casts_cache_dir_string_to_path(self) -> None:
|
||||
cache_dir = "/test_cache"
|
||||
encoder = OpenClipTextualEncoder("ViT-B-32__openai", cache_dir=cache_dir)
|
||||
|
@ -125,7 +134,7 @@ class TestBase:
|
|||
"immich-app/ViT-B-32__openai",
|
||||
cache_dir=encoder.cache_dir,
|
||||
local_dir=encoder.cache_dir,
|
||||
ignore_patterns=["*.armnn"],
|
||||
ignore_patterns=["*.armnn", "*.rknn"],
|
||||
)
|
||||
|
||||
def test_download_downloads_armnn_if_preferred_format(self, snapshot_download: mock.Mock) -> None:
|
||||
|
@ -136,7 +145,18 @@ class TestBase:
|
|||
"immich-app/ViT-B-32__openai",
|
||||
cache_dir=encoder.cache_dir,
|
||||
local_dir=encoder.cache_dir,
|
||||
ignore_patterns=[],
|
||||
ignore_patterns=["*.rknn"],
|
||||
)
|
||||
|
||||
def test_download_downloads_rknn_if_preferred_format(self, snapshot_download: mock.Mock) -> None:
|
||||
encoder = OpenClipTextualEncoder("ViT-B-32__openai", model_format=ModelFormat.RKNN)
|
||||
encoder.download()
|
||||
|
||||
snapshot_download.assert_called_once_with(
|
||||
"immich-app/ViT-B-32__openai",
|
||||
cache_dir=encoder.cache_dir,
|
||||
local_dir=encoder.cache_dir,
|
||||
ignore_patterns=["*.armnn"],
|
||||
)
|
||||
|
||||
def test_throws_exception_if_model_path_does_not_exist(
|
||||
|
@ -328,6 +348,36 @@ class TestAnnSession:
|
|||
np_spy.assert_has_calls([mock.call(input1), mock.call(input2)])
|
||||
|
||||
|
||||
class TestRknnSession:
|
||||
def test_creates_rknn_session(self, rknn_session: mock.Mock, info: mock.Mock, mocker: MockerFixture) -> None:
|
||||
model_path = mock.MagicMock(spec=Path)
|
||||
tpe = 1
|
||||
mocker.patch("app.sessions.rknn.soc_name", "rk3566")
|
||||
RknnSession(model_path)
|
||||
|
||||
rknn_session.assert_called_once_with(
|
||||
rknnModel=Path(str(model_path).replace("model", "rk3566")).as_posix(), tpes=tpe, func=runInference
|
||||
)
|
||||
|
||||
info.assert_has_calls(
|
||||
[mock.call(f"Loaded RKNN model from {str(model_path).replace('model','rk3566')} with {tpe} threads.")]
|
||||
)
|
||||
|
||||
def test_run_rknn(self, rknn_session: mock.Mock, mocker: MockerFixture) -> None:
|
||||
rknn_session.return_value.load.return_value = 123
|
||||
np_spy = mocker.spy(np, "ascontiguousarray")
|
||||
mocker.patch("app.sessions.rknn.soc_name", "rk3566")
|
||||
session = RknnSession(Path("ViT-B-32__openai"))
|
||||
[input1, input2] = [np.random.rand(1, 3, 224, 224).astype(np.float32) for _ in range(2)]
|
||||
input_feed = {"input.1": input1, "input.2": input2}
|
||||
|
||||
session.run(None, input_feed)
|
||||
|
||||
rknn_session.return_value.put.assert_called_once_with([input1, input2])
|
||||
np_spy.call_count == 2
|
||||
np_spy.assert_has_calls([mock.call(input1), mock.call(input2)])
|
||||
|
||||
|
||||
class TestCLIP:
|
||||
embedding = np.random.rand(512).astype(np.float32)
|
||||
cache_dir = Path("test_cache")
|
||||
|
|
90
machine-learning/poetry.lock
generated
90
machine-learning/poetry.lock
generated
|
@ -3047,6 +3047,94 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.1
|
|||
[package.extras]
|
||||
jupyter = ["ipywidgets (>=7.5.1,<9)"]
|
||||
|
||||
[[package]]
|
||||
name = "rknn-toolkit-lite2"
|
||||
version = "2.3.0"
|
||||
description = "Rockchip Neural Network Toolkit Lite2. (commit: 27e2f01)"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "rknn_toolkit_lite2-2.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b6733689bd09a262bcb6ba4744e690dd4b37ebeac4ed427cf45242c4b4ce9a4"},
|
||||
{file = "rknn_toolkit_lite2-2.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3e4fefe355dc34a155680e4bcb9e4abb37ebc271f045ec9e0a4a3a018bc5beb"},
|
||||
{file = "rknn_toolkit_lite2-2.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37394371d1561f470c553f39869d7c35ff93405dffe3d0d72babf297a2b0aee9"},
|
||||
{file = "rknn_toolkit_lite2-2.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1983c3bcb82402f64997a5b937427f75c9f4532387d780824c1b9d4bf2bd0c72"},
|
||||
{file = "rknn_toolkit_lite2-2.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89466e1e8aa6f887de0f9ae94fabbd6db461b8134b5020d12b4228fcf239132e"},
|
||||
{file = "rknn_toolkit_lite2-2.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:728f574be91df6c78019fd9241cf2a8331adf74f6bb807e33a0b45e20a1f31a0"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
numpy = "*"
|
||||
psutil = "*"
|
||||
"ruamel.yaml" = "*"
|
||||
|
||||
[[package]]
|
||||
name = "ruamel-yaml"
|
||||
version = "0.18.6"
|
||||
description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"},
|
||||
{file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""}
|
||||
|
||||
[package.extras]
|
||||
docs = ["mercurial (>5.7)", "ryd"]
|
||||
jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"]
|
||||
|
||||
[[package]]
|
||||
name = "ruamel-yaml-clib"
|
||||
version = "0.2.12"
|
||||
description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
files = [
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:11f891336688faf5156a36293a9c362bdc7c88f03a8a027c2c1d8e0bcde998e5"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a606ef75a60ecf3d924613892cc603b154178ee25abb3055db5062da811fd969"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd5415dded15c3822597455bc02bcd66e81ef8b7a48cb71a33628fc9fdde39df"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d84318609196d6bd6da0edfa25cedfbabd8dbde5140a0a23af29ad4b8f91fb1e"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb43a269eb827806502c7c8efb7ae7e9e9d0573257a46e8e952f4d4caba4f31e"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:943f32bc9dedb3abff9879edc134901df92cfce2c3d5c9348f172f62eb2d771d"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c3829bb364fdb8e0332c9931ecf57d9be3519241323c5274bd82f709cebc0c"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:e7e3736715fbf53e9be2a79eb4db68e4ed857017344d697e8b9749444ae57475"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7e75b4965e1d4690e93021adfcecccbca7d61c7bddd8e22406ef2ff20d74ef"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bc5f1e1c28e966d61d2519f2a3d451ba989f9ea0f2307de7bc45baa526de9e45"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a0e060aace4c24dcaf71023bbd7d42674e3b230f7e7b97317baf1e953e5b519"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"},
|
||||
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"},
|
||||
{file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.9.2"
|
||||
|
@ -3737,4 +3825,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.10,<4.0"
|
||||
content-hash = "b690d5fbd141da3947f4f1dc029aba1b95e7faafd723166f2c4bdc47a66c095e"
|
||||
content-hash = "0dfd0c31320434bf52aa454d7981faf8cbb162d23276bb05519ade9e8b6d6194"
|
||||
|
|
|
@ -59,6 +59,13 @@ optional = true
|
|||
[tool.poetry.group.armnn.dependencies]
|
||||
onnxruntime = "^1.15.0"
|
||||
|
||||
[tool.poetry.group.rknn]
|
||||
optional = true
|
||||
|
||||
[tool.poetry.group.rknn.dependencies]
|
||||
rknn-toolkit-lite2 = "^2.3.0"
|
||||
onnxruntime = "^1.15.0"
|
||||
|
||||
[[tool.poetry.source]]
|
||||
name = "cuda12"
|
||||
url = "https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/"
|
||||
|
|
0
machine-learning/rknn/__init__.py
Normal file
0
machine-learning/rknn/__init__.py
Normal file
53
machine-learning/rknn/export/build_rknn.py
Normal file
53
machine-learning/rknn/export/build_rknn.py
Normal file
|
@ -0,0 +1,53 @@
|
|||
import argparse
|
||||
import os
|
||||
|
||||
parser = argparse.ArgumentParser("RKNN model converting")
|
||||
parser.add_argument("model", help="Directory of the model that will be exported to RKNN ex:ViT-B-32__openai.", type=str)
|
||||
parser.add_argument("target_platform", help="target platform ex:rk3566", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def ConvertModel(model_path='ViT-B-32__openai/textual/model.onnx', target_platform='rk3566', dynamic_input = None):
|
||||
# E build: Repeat call the 'rknn.build' or 'rknn.hybrid_quantization_step1' is not allow!
|
||||
from rknn.api import RKNN
|
||||
rknn = RKNN(verbose=False)
|
||||
|
||||
rknn.config(target_platform=target_platform, dynamic_input=dynamic_input)
|
||||
ret = rknn.load_onnx(model=model_path)
|
||||
|
||||
if ret != 0:
|
||||
print("Load failed!")
|
||||
exit(ret)
|
||||
|
||||
ret = rknn.build(do_quantization=False)
|
||||
|
||||
if ret != 0:
|
||||
print("Build failed!")
|
||||
exit(ret)
|
||||
print(model_path.replace('model.onnx',f'{target_platform}.rknn'))
|
||||
ret = rknn.export_rknn(model_path.replace('model.onnx',f'{target_platform}.rknn'))
|
||||
if ret != 0:
|
||||
print('Export rknn model failed!')
|
||||
exit(ret)
|
||||
print('done')
|
||||
del rknn
|
||||
del RKNN
|
||||
|
||||
if not os.path.isfile(f'{model_path.replace("onnx","rknn")}'):
|
||||
print(f'Dummy model not found at {model_path.replace("onnx","rknn")}, creating one')
|
||||
with open(f'{model_path.replace("onnx","rknn")}', 'w'):
|
||||
pass
|
||||
|
||||
|
||||
if os.path.isdir(f'{args.model}/textual') and os.path.isdir(f'{args.model}/visual'): # is a clip model
|
||||
print('Converting Clip model.')
|
||||
ConvertModel(model_path=f'{args.model}/textual/model.onnx', target_platform=args.target_platform)
|
||||
ConvertModel(model_path=f'{args.model}/visual/model.onnx', target_platform=args.target_platform)
|
||||
|
||||
elif os.path.isdir(f'{args.model}/detection') and os.path.isdir(f'{args.model}/recognition'): # is a facial model
|
||||
print('Converting facial model.')
|
||||
ConvertModel(f'{args.model}/detection/model.onnx', args.target_platform, [[[1, 3, 640, 640]]])
|
||||
ConvertModel(f'{args.model}/recognition/model.onnx', args.target_platform, [[[1, 3, 112, 112]]])
|
||||
|
||||
else:
|
||||
print('Unknown model.')
|
13
machine-learning/rknn/export/convert.sh
Executable file
13
machine-learning/rknn/export/convert.sh
Executable file
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
|
||||
python3 build_rknn.py "$1" "$2" > immich_to_rknn2.log
|
||||
|
||||
# if "No lowering found for" found in log file, return error status 1
|
||||
if grep -q "No lowering found for" immich_to_rknn2.log; then
|
||||
echo -e "\e[31mSome operations are not supported by RKNN, please check the log file for details.\e[0m"
|
||||
exit 1
|
||||
else
|
||||
echo -e "\e[32mConversion completed successfully.\e[0m"
|
||||
rm immich_to_rknn2.log
|
||||
exit 0
|
||||
fi
|
1
machine-learning/rknn/export/requirements.txt
Normal file
1
machine-learning/rknn/export/requirements.txt
Normal file
|
@ -0,0 +1 @@
|
|||
rknn-toolkit2==2.3.0
|
83
machine-learning/rknn/rknnpool.py
Normal file
83
machine-learning/rknn/rknnpool.py
Normal file
|
@ -0,0 +1,83 @@
|
|||
# This code is from leafqycc/rknn-multi-threaded
|
||||
# Following Apache License 2.0
|
||||
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from queue import Queue
|
||||
import numpy as np
|
||||
from typing import Callable
|
||||
from numpy.typing import NDArray
|
||||
from app.config import log
|
||||
|
||||
supported_socs = ["rk3566", "rk3588"]
|
||||
coremask_supported_socs = ["rk3576","rk3588"]
|
||||
|
||||
try:
|
||||
from rknnlite.api import RKNNLite
|
||||
|
||||
with open("/proc/device-tree/compatible") as f:
|
||||
device_compatible_str = f.read()
|
||||
for soc in supported_socs:
|
||||
if soc in device_compatible_str:
|
||||
is_available = True
|
||||
soc_name = soc
|
||||
break
|
||||
else:
|
||||
is_available = False
|
||||
soc_name = None
|
||||
is_available = is_available and os.path.exists("/sys/kernel/debug/rknpu/load")
|
||||
except (FileNotFoundError, ImportError):
|
||||
log.debug("RKNN is not available")
|
||||
is_available = False
|
||||
soc_name = None
|
||||
|
||||
|
||||
def init_rknn(rknnModel) -> Callable:
|
||||
if not is_available:
|
||||
raise RuntimeError("rknn is not available!")
|
||||
rknn_lite = RKNNLite()
|
||||
ret = rknn_lite.load_rknn(rknnModel)
|
||||
if ret != 0:
|
||||
raise RuntimeError("Load RKNN rknnModel failed")
|
||||
|
||||
if soc_name in coremask_supported_socs:
|
||||
ret = rknn_lite.init_runtime(core_mask=RKNNLite.NPU_CORE_AUTO)
|
||||
else:
|
||||
ret = rknn_lite.init_runtime() # Please do not set this parameter on other platforms.
|
||||
|
||||
if ret != 0:
|
||||
raise RuntimeError("Init runtime environment failed")
|
||||
|
||||
return rknn_lite
|
||||
|
||||
|
||||
def init_rknns(rknnModel, tpes) -> list[Callable]:
|
||||
rknn_list = []
|
||||
for i in range(tpes):
|
||||
rknn_list.append(init_rknn(rknnModel))
|
||||
return rknn_list
|
||||
|
||||
|
||||
class RknnPoolExecutor:
|
||||
def __init__(self, rknnModel: str, tpes: int, func):
|
||||
self.tpes = tpes
|
||||
self.queue = Queue()
|
||||
self.rknn_pool = init_rknns(rknnModel, tpes)
|
||||
self.pool = ThreadPoolExecutor(max_workers=tpes)
|
||||
self.func = func
|
||||
self.num = 0
|
||||
|
||||
def put(self, inputs) -> None:
|
||||
self.queue.put(self.pool.submit(self.func, self.rknn_pool[self.num % self.tpes], inputs))
|
||||
self.num += 1
|
||||
|
||||
def get(self) -> list[list[NDArray[np.float32]], bool]:
|
||||
if self.queue.empty():
|
||||
return None
|
||||
fut = self.queue.get()
|
||||
return fut.result()
|
||||
|
||||
def release(self) -> None:
|
||||
self.pool.shutdown()
|
||||
for rknn_lite in self.rknn_pool:
|
||||
rknn_lite.release()
|
21
web/package-lock.json
generated
21
web/package-lock.json
generated
|
@ -16,6 +16,8 @@
|
|||
"@mdi/js": "^7.4.47",
|
||||
"@photo-sphere-viewer/core": "^5.11.5",
|
||||
"@photo-sphere-viewer/equirectangular-video-adapter": "^5.11.5",
|
||||
"@photo-sphere-viewer/resolution-plugin": "^5.11.5",
|
||||
"@photo-sphere-viewer/settings-plugin": "^5.11.5",
|
||||
"@photo-sphere-viewer/video-plugin": "^5.11.5",
|
||||
"@zoom-image/svelte": "^0.3.0",
|
||||
"dom-to-image": "^2.6.0",
|
||||
|
@ -1669,6 +1671,25 @@
|
|||
"@photo-sphere-viewer/video-plugin": "5.11.5"
|
||||
}
|
||||
},
|
||||
"node_modules/@photo-sphere-viewer/resolution-plugin": {
|
||||
"version": "5.11.5",
|
||||
"resolved": "https://registry.npmjs.org/@photo-sphere-viewer/resolution-plugin/-/resolution-plugin-5.11.5.tgz",
|
||||
"integrity": "sha512-Dbvp5bBtozD3IWt1Q0wORVaZBcB1bV9xUeoOS9A7F7b3EkQ2pkC5/jot/1AyM4wtU5wJ63NWHskQ1d7m6WWazQ==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@photo-sphere-viewer/core": "5.11.5",
|
||||
"@photo-sphere-viewer/settings-plugin": "5.11.5"
|
||||
}
|
||||
},
|
||||
"node_modules/@photo-sphere-viewer/settings-plugin": {
|
||||
"version": "5.11.5",
|
||||
"resolved": "https://registry.npmjs.org/@photo-sphere-viewer/settings-plugin/-/settings-plugin-5.11.5.tgz",
|
||||
"integrity": "sha512-ZgYaWjiBMhsoRH5ddW3h+v4J4LPmofsT7BBRq5UCssWw2Fsrvv7mFFRi4UbZ1qzeKmvNUOr8BaFQgX1ZLvUWfQ==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@photo-sphere-viewer/core": "5.11.5"
|
||||
}
|
||||
},
|
||||
"node_modules/@photo-sphere-viewer/video-plugin": {
|
||||
"version": "5.11.5",
|
||||
"resolved": "https://registry.npmjs.org/@photo-sphere-viewer/video-plugin/-/video-plugin-5.11.5.tgz",
|
||||
|
|
|
@ -72,6 +72,8 @@
|
|||
"@mdi/js": "^7.4.47",
|
||||
"@photo-sphere-viewer/core": "^5.11.5",
|
||||
"@photo-sphere-viewer/equirectangular-video-adapter": "^5.11.5",
|
||||
"@photo-sphere-viewer/resolution-plugin": "^5.11.5",
|
||||
"@photo-sphere-viewer/settings-plugin": "^5.11.5",
|
||||
"@photo-sphere-viewer/video-plugin": "^5.11.5",
|
||||
"@zoom-image/svelte": "^0.3.0",
|
||||
"dom-to-image": "^2.6.0",
|
||||
|
|
|
@ -24,7 +24,7 @@
|
|||
{:then [data, { default: PhotoSphereViewer }]}
|
||||
<PhotoSphereViewer
|
||||
panorama={data}
|
||||
originalImageUrl={isWebCompatibleImage(asset) ? getAssetOriginalUrl(asset.id) : undefined}
|
||||
originalPanorama={isWebCompatibleImage(asset) ? getAssetOriginalUrl(asset.id) : undefined}
|
||||
/>
|
||||
{:catch}
|
||||
{$t('errors.failed_to_load_asset')}
|
||||
|
|
|
@ -7,18 +7,21 @@
|
|||
type AdapterConstructor,
|
||||
type PluginConstructor,
|
||||
} from '@photo-sphere-viewer/core';
|
||||
import { SettingsPlugin } from '@photo-sphere-viewer/settings-plugin';
|
||||
import { ResolutionPlugin } from '@photo-sphere-viewer/resolution-plugin';
|
||||
import '@photo-sphere-viewer/core/index.css';
|
||||
import '@photo-sphere-viewer/settings-plugin/index.css';
|
||||
import { onDestroy, onMount } from 'svelte';
|
||||
|
||||
interface Props {
|
||||
panorama: string | { source: string };
|
||||
originalImageUrl?: string;
|
||||
originalPanorama?: string | { source: string };
|
||||
adapter?: AdapterConstructor | [AdapterConstructor, unknown];
|
||||
plugins?: (PluginConstructor | [PluginConstructor, unknown])[];
|
||||
navbar?: boolean;
|
||||
}
|
||||
|
||||
let { panorama, originalImageUrl, adapter = EquirectangularAdapter, plugins = [], navbar = false }: Props = $props();
|
||||
let { panorama, originalPanorama, adapter = EquirectangularAdapter, plugins = [], navbar = false }: Props = $props();
|
||||
|
||||
let container: HTMLDivElement | undefined = $state();
|
||||
let viewer: Viewer;
|
||||
|
@ -30,9 +33,33 @@
|
|||
|
||||
viewer = new Viewer({
|
||||
adapter,
|
||||
plugins,
|
||||
plugins: [
|
||||
SettingsPlugin,
|
||||
[
|
||||
ResolutionPlugin,
|
||||
{
|
||||
defaultResolution: $alwaysLoadOriginalFile && originalPanorama ? 'original' : 'default',
|
||||
resolutions: [
|
||||
{
|
||||
id: 'default',
|
||||
label: 'Default',
|
||||
panorama,
|
||||
},
|
||||
...(originalPanorama
|
||||
? [
|
||||
{
|
||||
id: 'original',
|
||||
label: 'Original',
|
||||
panorama: originalPanorama,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
},
|
||||
],
|
||||
...plugins,
|
||||
],
|
||||
container,
|
||||
panorama,
|
||||
touchmoveTwoFingers: false,
|
||||
mousewheelCtrlKey: false,
|
||||
navbar,
|
||||
|
@ -40,15 +67,14 @@
|
|||
maxFov: 120,
|
||||
fisheye: false,
|
||||
});
|
||||
const resolutionPlugin = viewer.getPlugin(ResolutionPlugin) as ResolutionPlugin;
|
||||
|
||||
if (originalImageUrl && !$alwaysLoadOriginalFile) {
|
||||
if (originalPanorama && !$alwaysLoadOriginalFile) {
|
||||
const zoomHandler = ({ zoomLevel }: events.ZoomUpdatedEvent) => {
|
||||
// zoomLevel range: [0, 100]
|
||||
if (Math.round(zoomLevel) >= 75) {
|
||||
// Replace the preview with the original
|
||||
viewer.setPanorama(originalImageUrl, { showLoader: false, speed: 150 }).catch(() => {
|
||||
viewer.setPanorama(panorama, { showLoader: false, speed: 0 }).catch(() => {});
|
||||
});
|
||||
void resolutionPlugin.setResolution('original');
|
||||
viewer.removeEventListener(events.ZoomUpdatedEvent.type, zoomHandler);
|
||||
}
|
||||
};
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
<script lang="ts">
|
||||
import { getAssetOriginalUrl } from '$lib/utils';
|
||||
import { getAssetPlaybackUrl, getAssetOriginalUrl } from '$lib/utils';
|
||||
import { fade } from 'svelte/transition';
|
||||
import LoadingSpinner from '../shared-components/loading-spinner.svelte';
|
||||
import { t } from 'svelte-i18n';
|
||||
|
@ -22,7 +22,13 @@
|
|||
{#await modules}
|
||||
<LoadingSpinner />
|
||||
{:then [PhotoSphereViewer, adapter, videoPlugin]}
|
||||
<PhotoSphereViewer panorama={{ source: getAssetOriginalUrl(assetId) }} plugins={[videoPlugin]} {adapter} navbar />
|
||||
<PhotoSphereViewer
|
||||
panorama={{ source: getAssetPlaybackUrl(assetId) }}
|
||||
originalPanorama={{ source: getAssetOriginalUrl(assetId) }}
|
||||
plugins={[videoPlugin]}
|
||||
{adapter}
|
||||
navbar
|
||||
/>
|
||||
{:catch}
|
||||
{$t('errors.failed_to_load_asset')}
|
||||
{/await}
|
||||
|
|
Loading…
Reference in a new issue