1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-02-03 01:22:44 +01:00

Compare commits

...

85 commits

Author SHA1 Message Date
Yoni Yang
209a67c312
Merge 6e08f1f371 into c2ba1cc202 2025-01-28 22:09:26 +05:00
Miguel Angel Nubla
c2ba1cc202
docs: add immich-upload-optimizer to Community Projects list () 2025-01-28 09:40:00 -06:00
PastLeo
08db77db23
feat: resolution selection and default preview playback for 360° panorama videos ()
* original/preview switching in photo-sphere-viewer

1. default to preview in photo-sphere-viewer video mode
2. install and integrate @photo-sphere-viewer/settings-plugin & @photo-sphere-viewer/resolution-plugin

* fix lint errors
2025-01-28 09:09:40 -06:00
yoni13
6e08f1f371 dont stuck on core_0 on rk3588 but untested 2025-01-25 19:14:34 +08:00
Yoni Yang
3ed5d3dbac
Merge branch 'main' into rknn-toolkit-lite2 2025-01-24 16:49:10 +08:00
Yoni Yang
f9387d8478
Merge branch 'main' into rknn-toolkit-lite2 2025-01-24 12:20:29 +08:00
Yoni Yang
4bb98471f4
Merge branch 'main' into rknn-toolkit-lite2 2025-01-23 23:22:24 +08:00
Yoni Yang
794da29411
Merge branch 'main' into rknn-toolkit-lite2 2025-01-22 11:43:59 +08:00
yoni13
dd52c2d68c Update permission 2025-01-19 23:19:44 +08:00
Yoni Yang
59e4b6598e
Merge branch 'main' into rknn-toolkit-lite2 2025-01-19 21:50:47 +08:00
yoni13
20ba9f97e9 update mapping 2025-01-19 13:15:38 +00:00
yoni13
ac4ce3ea9c add input outputs 2025-01-19 12:10:01 +00:00
yoni13
2b967ca358 raise NotImplementedError for now 2025-01-19 12:02:08 +08:00
yoni13
1653cd9cd7 update supported SOCs 2025-01-18 17:38:44 +08:00
yoni13
32f3707e52 fix types and ignored pattern 2025-01-18 17:17:48 +08:00
yoni13
58f1cc92d7 prettier happy 2025-01-18 17:03:49 +08:00
yoni13
d2b7e10f55 shellcheck happy 2025-01-18 16:56:15 +08:00
yoni13
b3ae5d34cc fix typo in tests 2025-01-18 16:54:18 +08:00
Yoni Yang
4e42fbc091
Merge branch 'main' into rknn-toolkit-lite2 2025-01-18 16:45:38 +08:00
yoni13
9926045d5e add a simple script to notify user if some op is not supported 2025-01-18 16:32:08 +08:00
yoni13
d7381ab5c1 refactor ignore_patterns 2025-01-18 04:48:11 +00:00
yoni13
87a46dcc5e remove unnecessary print 2025-01-18 11:30:49 +08:00
yoni13
be76857ae6 make these functions snake case. 2025-01-18 11:03:54 +08:00
yoni13
f5de3de163 fix typo and add a propper var name 2025-01-18 10:46:57 +08:00
yoni13
3634ae1f5b fix granularity 2025-01-18 09:58:39 +08:00
yoni13
05675921be remove unrequired devices 2025-01-17 20:09:50 +08:00
yoni13
f067212491 tpe 2025-01-17 19:56:23 +08:00
yoni13
bc48b67379 switch to sha256 2025-01-17 19:42:23 +08:00
yoni13
26d5fb0ac6 add checksum for libnnrt.so 2025-01-17 19:39:26 +08:00
yoni13
f32d991131 changes some cases 2025-01-17 19:25:01 +08:00
yoni13
9882b83cd4 Should FIx the quote that made mypy unhappy 2025-01-15 00:35:43 +08:00
yoni13
01eb09526e trying to fix pytest 2025-01-15 00:28:48 +08:00
yoni13
b5a4ed5160 this duplicated? 2025-01-14 19:22:28 +08:00
yoni13
0f03f77e8e remove non implemented tests 2025-01-14 19:02:16 +08:00
yoni13
c21ce40d9c switch to Runtime error instead of exit() 2025-01-14 18:50:21 +08:00
Yoni Yang
cb01a11f19
Merge branch 'main' into rknn-toolkit-lite2 2025-01-14 18:44:31 +08:00
yoni13
5244ed6d4d black app export 2025-01-14 18:40:28 +08:00
yoni13
8b80d034cb fixed some bugs 2025-01-14 10:38:45 +00:00
yoni13
4b0f93cf6a add test,founds bugs, fix it tomorrow 2025-01-14 01:08:44 +08:00
yoni13
6c4e6cb96f reformat 2025-01-13 18:37:25 +08:00
yoni13
b6cc2054c5 ignore rknn model if not using it 2025-01-13 18:37:01 +08:00
Yoni Yang
f328104e84
Merge branch 'main' into rknn-toolkit-lite2 2025-01-13 18:33:05 +08:00
yoni13
2f7e44aa63 typing be happy. 2025-01-13 18:24:12 +08:00
yoni13
ebdfe1b7b6 Load model by SOC name 2025-01-13 17:08:16 +08:00
yoni13
daf886088a Add export script 2025-01-13 05:44:22 +00:00
yoni13
4c7ac1438b only load knnx model when required 2025-01-12 19:11:16 +00:00
Yoni Yang
8965a9fb16
Merge branch 'main' into rknn-toolkit-lite2 2025-01-13 01:41:55 +08:00
yoni13
7ae4b7129d format be happy 2025-01-13 01:40:20 +08:00
yoni13
1775397a84 Sort them by alphablet 2025-01-13 01:38:14 +08:00
yoni13
68fccad462 Fix docs. 2025-01-13 01:11:32 +08:00
yoni13
bb67a9db6e fix formatting 2025-01-13 00:50:49 +08:00
yoni13
c109e28686 DOCS 2025-01-13 00:44:22 +08:00
yoni13
e6ff21b345 set default thread num to 2, not everyone has 8 gigs of ram 2025-01-12 16:05:18 +08:00
yoni13
c665fd2625 Fix Please do not set this parameter on other platforms. 2025-01-12 01:29:24 +08:00
yoni13
c72cf61ed0 support core_mask for specfic socs 2025-01-12 01:24:24 +08:00
yoni13
19ee48f6f0 fix path 2025-01-12 01:09:36 +08:00
yoni13
efaf70eb9d Set running threads from env 2025-01-12 01:02:16 +08:00
yoni13
665718b09e add rknn to src 2025-01-11 21:11:30 +08:00
yoni13
807111e3b5 Should Fix No module named 'rknn' 2025-01-11 20:41:38 +08:00
yoni13
815ed1ae66 Install onnxruntime 2025-01-11 20:33:48 +08:00
yoni13
416211916d Check if NPU drivers is loaded or not. 2025-01-11 17:59:16 +08:00
yoni13
23d0ea0e7b ruff 2025-01-11 16:28:26 +08:00
yoni13
d5e453a773 ruff format 2025-01-11 16:26:17 +08:00
yoni13
7f2af6f819 Fix typo: rknnlite.api 2025-01-11 16:19:51 +08:00
yoni13
f4671f4886 Indentation issue 2025-01-11 16:16:26 +08:00
yoni13
7aaf3aa57b Remove unused imports. 2025-01-11 16:03:33 +08:00
yoni13
506ca0d3a4 Dockerfile for rknn 2025-01-11 15:47:24 +08:00
yoni13
d5ef821b24 Set group RKNN to optional 2025-01-11 15:30:05 +08:00
yoni13
d10147f478 Handling Import and file not found Error for non-arm devices. 2025-01-11 15:19:53 +08:00
Yoni Yang
66004e3b83
Merge branch 'immich-app:main' into rknn-toolkit-lite2 2025-01-11 10:45:55 +08:00
yoni13
c20d110257 support for rknn.rknnpool.is_available 2025-01-11 10:39:45 +08:00
yoni13
a2722e16e7 Revert my changes to dockerfiles 2025-01-11 10:13:03 +08:00
yoni13
4d704e9f73 fix inf,-inf with 2 concurrency 2025-01-10 14:04:18 +00:00
Yoni Yang
9bc3e5b2e2
Update rknn.py 2025-01-10 20:20:21 +08:00
Yoni Yang
8608b9c6c5
Merge branch 'immich-app:main' into rknn-toolkit-lite2 2025-01-09 18:40:40 +08:00
yoni13
a94fad543b all infrencing works with 1 max job concurrency 2025-01-09 10:38:40 +00:00
Yoni Yang
082c426e34
Merge branch 'immich-app:main' into rknn-toolkit-lite2 2024-12-25 20:24:17 +08:00
Yoni Yang
da152bd284
Merge branch 'immich-app:main' into rknn-toolkit-lite2 2024-12-13 13:46:24 +08:00
Yoni Yang
257cc6c963 Init commit for using rknn, RecognitionFormDataLoadTest doesnt work 2024-12-04 14:32:46 +00:00
Yoni Yang
4140e93aea
Merge branch 'immich-app:main' into rknn-toolkit-lite2 2024-12-04 13:15:14 +08:00
Yoni Yang
b6c4b37237
Merge branch 'immich-app:main' into rknn-toolkit2 2024-12-03 06:00:43 -08:00
Yoni Yang
bc849e2e9f ViT-B-32__openai/textual/ Runs with emulator now. 2024-12-01 16:42:53 +00:00
Yoni Yang
7fddf282cf lowercase 2024-11-30 14:24:38 +00:00
Yoni Yang
6ffc227330 test 2024-11-29 07:43:59 +00:00
Yoni Yang
8ef3e49f74 untested 2024-11-29 07:42:09 +00:00
28 changed files with 513 additions and 52 deletions

View file

@ -48,7 +48,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
suffix: ["", "-cuda", "-openvino", "-armnn"]
suffix: ["", "-cuda", "-openvino", "-armnn","-rknn"]
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
@ -116,6 +116,9 @@ jobs:
- platforms: linux/arm64
device: armnn
suffix: -armnn
- platforms: linux/arm64
device: rknn
suffix: -rknn
steps:
- name: Checkout
@ -307,4 +310,4 @@ jobs:
run: exit 1
- name: All jobs passed or skipped
if: ${{ !(contains(needs.*.result, 'failure')) }}
run: echo "All jobs passed or skipped" && echo "${{ toJSON(needs.*.result) }}"
run: echo "All jobs passed or skipped" && echo "${{ toJSON(needs.*.result) }}"

View file

@ -390,7 +390,7 @@ jobs:
poetry run black --check app export
- name: Run mypy type checking
run: |
poetry run mypy --install-types --non-interactive --strict app/
mkdir .mypy_cache && poetry run mypy --install-types --non-interactive --strict app/
- name: Run tests and coverage
run: |
poetry run pytest app --cov=app --cov-report term-missing

View file

@ -86,12 +86,12 @@ services:
image: immich-machine-learning-dev:latest
# extends:
# file: hwaccel.ml.yml
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference
build:
context: ../machine-learning
dockerfile: Dockerfile
args:
- DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
- DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference
ports:
- 3003:3003
volumes:

View file

@ -29,12 +29,12 @@ services:
image: immich-machine-learning:latest
# extends:
# file: hwaccel.ml.yml
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference
build:
context: ../machine-learning
dockerfile: Dockerfile
args:
- DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
- DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference
ports:
- 3003:3003
volumes:
@ -68,22 +68,12 @@ services:
- 5432:5432
healthcheck:
test: >-
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
--command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
echo "checksum failure count is $$Chksum";
[ "$$Chksum" = '0' ] || exit 1
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1; Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1
interval: 5m
start_interval: 30s
start_period: 5m
command: >-
postgres
-c shared_preload_libraries=vectors.so
-c 'search_path="$$user", public, vectors'
-c logging_collector=on
-c max_wal_size=2GB
-c shared_buffers=512MB
-c wal_compression=on
postgres -c shared_preload_libraries=vectors.so -c 'search_path="$$user", public, vectors' -c logging_collector=on -c max_wal_size=2GB -c shared_buffers=512MB -c wal_compression=on
restart: always
# set IMMICH_TELEMETRY_INCLUDE=all in .env to enable metrics
@ -100,7 +90,7 @@ services:
# add data source for http://immich-prometheus:9090 to get started
immich-grafana:
container_name: immich_grafana
command: ['./run.sh', '-disable-reporting']
command: [ './run.sh', '-disable-reporting' ]
ports:
- 3000:3000
image: grafana/grafana:11.4.0-ubuntu@sha256:afccec22ba0e4815cca1d2bf3836e414322390dc78d77f1851976ffa8d61051c

View file

@ -32,12 +32,12 @@ services:
immich-machine-learning:
container_name: immich_machine_learning
# For hardware acceleration, add one of -[armnn, cuda, openvino] to the image tag.
# For hardware acceleration, add one of -[armnn, cuda, openvino, rknn] to the image tag.
# Example tag: ${IMMICH_VERSION:-release}-cuda
image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release}
# extends: # uncomment this section for hardware acceleration - see https://immich.app/docs/features/ml-hardware-acceleration
# file: hwaccel.ml.yml
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference - use the `-wsl` version for WSL2 where applicable
# service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl, rknn] for accelerated inference - use the `-wsl` version for WSL2 where applicable
volumes:
- model-cache:/cache
env_file:
@ -66,22 +66,12 @@ services:
- ${DB_DATA_LOCATION}:/var/lib/postgresql/data
healthcheck:
test: >-
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
--command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
echo "checksum failure count is $$Chksum";
[ "$$Chksum" = '0' ] || exit 1
pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1; Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1
interval: 5m
start_interval: 30s
start_period: 5m
command: >-
postgres
-c shared_preload_libraries=vectors.so
-c 'search_path="$$user", public, vectors'
-c logging_collector=on
-c max_wal_size=2GB
-c shared_buffers=512MB
-c wal_compression=on
postgres -c shared_preload_libraries=vectors.so -c 'search_path="$$user", public, vectors' -c logging_collector=on -c max_wal_size=2GB -c shared_buffers=512MB -c wal_compression=on
restart: always
volumes:

View file

@ -13,6 +13,14 @@ services:
volumes:
- /lib/firmware/mali_csffw.bin:/lib/firmware/mali_csffw.bin:ro # Mali firmware for your chipset (not always required depending on the driver)
- /usr/lib/libmali.so:/usr/lib/libmali.so:ro # Mali driver for your chipset (always required)
rknn:
security_opt:
- systempaths=unconfined
devices:
- /dev/dri:/dev/dri
volumes:
- /sys/kernel/debug/:/sys/kernel/debug/:ro
cpu: {}

View file

@ -12,6 +12,7 @@ You do not need to redo any machine learning jobs after enabling hardware accele
- ARM NN (Mali)
- CUDA (NVIDIA GPUs with [compute capability](https://developer.nvidia.com/cuda-gpus) 5.2 or higher)
- OpenVINO (Intel discrete GPUs such as Iris Xe and Arc)
- RKNN (Rockchip)
## Limitations
@ -46,6 +47,15 @@ You do not need to redo any machine learning jobs after enabling hardware accele
- The server must have a discrete GPU, i.e. Iris Xe or Arc. Expect issues when attempting to use integrated graphics.
- Ensure the server's kernel version is new enough to use the device for hardware accceleration.
#### RKNN
- You must have a supported Rockchip SoC, only RK3566 and RK3588 are supported at this moment.
- Make sure you have the appropriate linux kernel driver installed
- This is usually pre-installed on the device vendor's Linux images
- RKNPU driver V0.9.8 or later must be available in the host server
- You may confirm this by running `cat /sys/kernel/debug/rknpu/version` to check the version
- Optional: Configure your `.env` file, see [environment variables](/docs/install/environment-variables) for RKNN specific settings
## Setup
1. If you do not already have it, download the latest [`hwaccel.ml.yml`][hw-file] file and ensure it's in the same folder as the `docker-compose.yml`.

View file

@ -168,6 +168,8 @@ Redis (Sentinel) URL example JSON before encoding:
| `MACHINE_LEARNING_ANN_TUNING_LEVEL` | ARM-NN GPU tuning level (1: rapid, 2: normal, 3: exhaustive) | `2` | machine learning |
| `MACHINE_LEARNING_DEVICE_IDS`<sup>\*4</sup> | Device IDs to use in multi-GPU environments | `0` | machine learning |
| `MACHINE_LEARNING_MAX_BATCH_SIZE__FACIAL_RECOGNITION` | Set the maximum number of faces that will be processed at once by the facial recognition model | None (`1` if using OpenVINO) | machine learning |
| `MACHINE_LEARNING_RKNN` | Enable RKNN hardware acceleration if supported | `True` | machine learning |
| `MACHINE_LEARNING_RKNN_THREADS` | How many threads of RKNN runtime should be spinned up while inferencing. | `1` | machine learning |
\*1: It is recommended to begin with this parameter when changing the concurrency levels of the machine learning service and then tune the other ones.

View file

@ -99,6 +99,11 @@ const projects: CommunityProjectProps[] = [
description: 'Downloads a configurable number of random photos based on people or album ID.',
url: 'https://github.com/jon6fingrs/immich-dl',
},
{
title: 'Immich Upload Optimizer',
description: 'Automatically optimize files uploaded to Immich in order to save storage space',
url: 'https://github.com/miguelangel-nubla/immich-upload-optimizer',
},
];
function CommunityProject({ title, description, url }: CommunityProjectProps): JSX.Element {

View file

@ -15,6 +15,8 @@ RUN mkdir /opt/armnn && \
cd /opt/ann && \
sh build.sh
FROM builder-cpu AS builder-rknn
FROM builder-${DEVICE} AS builder
ARG DEVICE
@ -80,6 +82,10 @@ COPY --from=builder-armnn \
/opt/ann/build.sh \
/opt/armnn/
FROM prod-cpu AS prod-rknn
ADD --checksum=sha256:73993ed4b440460825f21611731564503cc1d5a0c123746477da6cd574f34885 https://github.com/airockchip/rknn-toolkit2/raw/refs/tags/v2.3.0/rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so /usr/lib/
FROM prod-${DEVICE} AS prod
ARG DEVICE
@ -104,9 +110,10 @@ RUN echo "hard core 0" >> /etc/security/limits.conf && \
COPY --from=builder /opt/venv /opt/venv
COPY ann/ann.py /usr/src/ann/ann.py
COPY rknn/rknnpool.py /usr/src/rknn/rknnpool.py
COPY start.sh log_conf.json gunicorn_conf.py ./
COPY app .
ENTRYPOINT ["tini", "--"]
CMD ["./start.sh"]
HEALTHCHECK CMD python3 healthcheck.py
HEALTHCHECK CMD python3 healthcheck.py

View file

@ -64,6 +64,8 @@ class Settings(BaseSettings):
ann: bool = True
ann_fp16_turbo: bool = False
ann_tuning_level: int = 2
rknn: bool = True
rknn_threads: int = 1
preload: PreloadModelData | None = None
max_batch_size: MaxBatchSize | None = None

View file

@ -136,6 +136,12 @@ def ann_session() -> Iterator[mock.Mock]:
yield mocked
@pytest.fixture(scope="function")
def rknn_session() -> Iterator[mock.Mock]:
with mock.patch("app.sessions.rknn.RknnPoolExecutor") as mocked:
yield mocked
@pytest.fixture(scope="function")
def rmtree() -> Iterator[mock.Mock]:
with mock.patch("app.models.base.rmtree", autospec=True) as mocked:

View file

@ -8,7 +8,9 @@ from typing import Any, ClassVar
from huggingface_hub import snapshot_download
import ann.ann
import rknn.rknnpool
from app.sessions.ort import OrtSession
from app.sessions.rknn import RknnSession
from ..config import clean_name, log, settings
from ..schemas import ModelFormat, ModelIdentity, ModelSession, ModelTask, ModelType
@ -66,12 +68,17 @@ class InferenceModel(ABC):
pass
def _download(self) -> None:
ignore_patterns = [] if self.model_format == ModelFormat.ARMNN else ["*.armnn"]
ignored_patterns: dict[ModelFormat, list[str]] = {
ModelFormat.ONNX: ["*.armnn", "*.rknn"],
ModelFormat.ARMNN: ["*.rknn"],
ModelFormat.RKNN: ["*.armnn"],
}
snapshot_download(
f"immich-app/{clean_name(self.model_name)}",
cache_dir=self.cache_dir,
local_dir=self.cache_dir,
ignore_patterns=ignore_patterns,
ignore_patterns=ignored_patterns.get(self.model_format, []),
)
def _load(self) -> ModelSession:
@ -108,6 +115,8 @@ class InferenceModel(ABC):
session: ModelSession = AnnSession(model_path)
case ".onnx":
session = OrtSession(model_path)
case ".rknn":
session = RknnSession(model_path)
case _:
raise ValueError(f"Unsupported model file type: {model_path.suffix}")
return session
@ -155,4 +164,9 @@ class InferenceModel(ABC):
@property
def _model_format_default(self) -> ModelFormat:
return ModelFormat.ARMNN if ann.ann.is_available and settings.ann else ModelFormat.ONNX
if rknn.rknnpool.is_available and settings.rknn:
return ModelFormat.RKNN
elif ann.ann.is_available and settings.ann:
return ModelFormat.ARMNN
else:
return ModelFormat.ONNX

View file

@ -35,6 +35,7 @@ class ModelType(StrEnum):
class ModelFormat(StrEnum):
ARMNN = "armnn"
ONNX = "onnx"
RKNN = "rknn"
class ModelSource(StrEnum):

View file

@ -0,0 +1,73 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, NamedTuple
import numpy as np
from numpy.typing import NDArray
from app.schemas import SessionNode
from rknn.rknnpool import RknnPoolExecutor, soc_name
from ..config import log, settings
def runInference(rknn_lite: Any, input: list[NDArray[np.float32]]) -> list[NDArray[np.float32]]:
outputs: list[NDArray[np.float32]] = rknn_lite.inference(inputs=input, data_format="nchw")
return outputs
input_output_mapping: dict[str, dict[str, Any]] = {
"detection": {
"input": {"norm_tensor:0": (1, 3, 640, 640)},
"output": {
"norm_tensor:1": (12800, 1),
"norm_tensor:2": (3200, 1),
"norm_tensor:3": (800, 1),
"norm_tensor:4": (12800, 4),
"norm_tensor:5": (3200, 4),
"norm_tensor:6": (800, 4),
"norm_tensor:7": (12800, 10),
"norm_tensor:8": (3200, 10),
"norm_tensor:9": (800, 10),
},
},
"recognition": {"input": {"norm_tensor:0": (1, 3, 112, 112)}, "output": {"norm_tensor:1": (1, 512)}},
}
class RknnSession:
def __init__(self, model_path: Path | str):
self.model_path = Path(str(model_path).replace("model", soc_name))
self.model_type = "detection" if "detection" in self.model_path.as_posix() else "recognition"
self.tpe = settings.rknn_threads
log.info(f"Loading RKNN model from {self.model_path} with {self.tpe} threads.")
self.rknnpool = RknnPoolExecutor(rknnModel=self.model_path.as_posix(), tpes=self.tpe, func=runInference)
log.info(f"Loaded RKNN model from {self.model_path} with {self.tpe} threads.")
def __del__(self) -> None:
self.rknnpool.release()
def get_inputs(self) -> list[SessionNode]:
return [RknnNode(name=k, shape=v) for k, v in input_output_mapping[self.model_type]["input"].items()]
def get_outputs(self) -> list[SessionNode]:
return [RknnNode(name=k, shape=v) for k, v in input_output_mapping[self.model_type]["output"].items()]
def run(
self,
output_names: list[str] | None,
input_feed: dict[str, NDArray[np.float32]] | dict[str, NDArray[np.int32]],
run_options: Any = None,
) -> list[NDArray[np.float32]]:
input_data: list[NDArray[np.float32]] = [np.ascontiguousarray(v) for v in input_feed.values()]
self.rknnpool.put(input_data)
outputs: list[NDArray[np.float32]] = self.rknnpool.get()
return outputs
class RknnNode(NamedTuple):
name: str | None
shape: tuple[int, ...]

View file

@ -25,6 +25,7 @@ from app.models.facial_recognition.detection import FaceDetector
from app.models.facial_recognition.recognition import FaceRecognizer
from app.sessions.ann import AnnSession
from app.sessions.ort import OrtSession
from app.sessions.rknn import RknnSession, runInference
from .config import Settings, settings
from .models.base import InferenceModel
@ -69,6 +70,14 @@ class TestBase:
assert encoder.model_format == ModelFormat.ARMNN
def test_sets_default_model_format_rknn(self, mocker: MockerFixture) -> None:
mocker.patch.object(settings, "rknn", True)
mocker.patch("rknn.rknnpool.is_available", False)
encoder = OpenClipTextualEncoder("ViT-B-32__openai")
assert encoder.model_format == ModelFormat.ONNX
def test_casts_cache_dir_string_to_path(self) -> None:
cache_dir = "/test_cache"
encoder = OpenClipTextualEncoder("ViT-B-32__openai", cache_dir=cache_dir)
@ -125,7 +134,7 @@ class TestBase:
"immich-app/ViT-B-32__openai",
cache_dir=encoder.cache_dir,
local_dir=encoder.cache_dir,
ignore_patterns=["*.armnn"],
ignore_patterns=["*.armnn", "*.rknn"],
)
def test_download_downloads_armnn_if_preferred_format(self, snapshot_download: mock.Mock) -> None:
@ -136,7 +145,18 @@ class TestBase:
"immich-app/ViT-B-32__openai",
cache_dir=encoder.cache_dir,
local_dir=encoder.cache_dir,
ignore_patterns=[],
ignore_patterns=["*.rknn"],
)
def test_download_downloads_rknn_if_preferred_format(self, snapshot_download: mock.Mock) -> None:
encoder = OpenClipTextualEncoder("ViT-B-32__openai", model_format=ModelFormat.RKNN)
encoder.download()
snapshot_download.assert_called_once_with(
"immich-app/ViT-B-32__openai",
cache_dir=encoder.cache_dir,
local_dir=encoder.cache_dir,
ignore_patterns=["*.armnn"],
)
def test_throws_exception_if_model_path_does_not_exist(
@ -328,6 +348,36 @@ class TestAnnSession:
np_spy.assert_has_calls([mock.call(input1), mock.call(input2)])
class TestRknnSession:
def test_creates_rknn_session(self, rknn_session: mock.Mock, info: mock.Mock, mocker: MockerFixture) -> None:
model_path = mock.MagicMock(spec=Path)
tpe = 1
mocker.patch("app.sessions.rknn.soc_name", "rk3566")
RknnSession(model_path)
rknn_session.assert_called_once_with(
rknnModel=Path(str(model_path).replace("model", "rk3566")).as_posix(), tpes=tpe, func=runInference
)
info.assert_has_calls(
[mock.call(f"Loaded RKNN model from {str(model_path).replace('model','rk3566')} with {tpe} threads.")]
)
def test_run_rknn(self, rknn_session: mock.Mock, mocker: MockerFixture) -> None:
rknn_session.return_value.load.return_value = 123
np_spy = mocker.spy(np, "ascontiguousarray")
mocker.patch("app.sessions.rknn.soc_name", "rk3566")
session = RknnSession(Path("ViT-B-32__openai"))
[input1, input2] = [np.random.rand(1, 3, 224, 224).astype(np.float32) for _ in range(2)]
input_feed = {"input.1": input1, "input.2": input2}
session.run(None, input_feed)
rknn_session.return_value.put.assert_called_once_with([input1, input2])
np_spy.call_count == 2
np_spy.assert_has_calls([mock.call(input1), mock.call(input2)])
class TestCLIP:
embedding = np.random.rand(512).astype(np.float32)
cache_dir = Path("test_cache")

View file

@ -3047,6 +3047,94 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.1
[package.extras]
jupyter = ["ipywidgets (>=7.5.1,<9)"]
[[package]]
name = "rknn-toolkit-lite2"
version = "2.3.0"
description = "Rockchip Neural Network Toolkit Lite2. (commit: 27e2f01)"
optional = false
python-versions = "*"
files = [
{file = "rknn_toolkit_lite2-2.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b6733689bd09a262bcb6ba4744e690dd4b37ebeac4ed427cf45242c4b4ce9a4"},
{file = "rknn_toolkit_lite2-2.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3e4fefe355dc34a155680e4bcb9e4abb37ebc271f045ec9e0a4a3a018bc5beb"},
{file = "rknn_toolkit_lite2-2.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37394371d1561f470c553f39869d7c35ff93405dffe3d0d72babf297a2b0aee9"},
{file = "rknn_toolkit_lite2-2.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1983c3bcb82402f64997a5b937427f75c9f4532387d780824c1b9d4bf2bd0c72"},
{file = "rknn_toolkit_lite2-2.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89466e1e8aa6f887de0f9ae94fabbd6db461b8134b5020d12b4228fcf239132e"},
{file = "rknn_toolkit_lite2-2.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:728f574be91df6c78019fd9241cf2a8331adf74f6bb807e33a0b45e20a1f31a0"},
]
[package.dependencies]
numpy = "*"
psutil = "*"
"ruamel.yaml" = "*"
[[package]]
name = "ruamel-yaml"
version = "0.18.6"
description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order"
optional = false
python-versions = ">=3.7"
files = [
{file = "ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636"},
{file = "ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b"},
]
[package.dependencies]
"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""}
[package.extras]
docs = ["mercurial (>5.7)", "ryd"]
jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"]
[[package]]
name = "ruamel-yaml-clib"
version = "0.2.12"
description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml"
optional = false
python-versions = ">=3.9"
files = [
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:11f891336688faf5156a36293a9c362bdc7c88f03a8a027c2c1d8e0bcde998e5"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a606ef75a60ecf3d924613892cc603b154178ee25abb3055db5062da811fd969"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd5415dded15c3822597455bc02bcd66e81ef8b7a48cb71a33628fc9fdde39df"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"},
{file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d84318609196d6bd6da0edfa25cedfbabd8dbde5140a0a23af29ad4b8f91fb1e"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb43a269eb827806502c7c8efb7ae7e9e9d0573257a46e8e952f4d4caba4f31e"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"},
{file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:943f32bc9dedb3abff9879edc134901df92cfce2c3d5c9348f172f62eb2d771d"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c3829bb364fdb8e0332c9931ecf57d9be3519241323c5274bd82f709cebc0c"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"},
{file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:e7e3736715fbf53e9be2a79eb4db68e4ed857017344d697e8b9749444ae57475"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7e75b4965e1d4690e93021adfcecccbca7d61c7bddd8e22406ef2ff20d74ef"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"},
{file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bc5f1e1c28e966d61d2519f2a3d451ba989f9ea0f2307de7bc45baa526de9e45"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a0e060aace4c24dcaf71023bbd7d42674e3b230f7e7b97317baf1e953e5b519"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"},
{file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"},
{file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"},
]
[[package]]
name = "ruff"
version = "0.9.2"
@ -3737,4 +3825,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<4.0"
content-hash = "b690d5fbd141da3947f4f1dc029aba1b95e7faafd723166f2c4bdc47a66c095e"
content-hash = "0dfd0c31320434bf52aa454d7981faf8cbb162d23276bb05519ade9e8b6d6194"

View file

@ -59,6 +59,13 @@ optional = true
[tool.poetry.group.armnn.dependencies]
onnxruntime = "^1.15.0"
[tool.poetry.group.rknn]
optional = true
[tool.poetry.group.rknn.dependencies]
rknn-toolkit-lite2 = "^2.3.0"
onnxruntime = "^1.15.0"
[[tool.poetry.source]]
name = "cuda12"
url = "https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/"

View file

View file

@ -0,0 +1,53 @@
import argparse
import os
parser = argparse.ArgumentParser("RKNN model converting")
parser.add_argument("model", help="Directory of the model that will be exported to RKNN ex:ViT-B-32__openai.", type=str)
parser.add_argument("target_platform", help="target platform ex:rk3566", type=str)
args = parser.parse_args()
def ConvertModel(model_path='ViT-B-32__openai/textual/model.onnx', target_platform='rk3566', dynamic_input = None):
# E build: Repeat call the 'rknn.build' or 'rknn.hybrid_quantization_step1' is not allow!
from rknn.api import RKNN
rknn = RKNN(verbose=False)
rknn.config(target_platform=target_platform, dynamic_input=dynamic_input)
ret = rknn.load_onnx(model=model_path)
if ret != 0:
print("Load failed!")
exit(ret)
ret = rknn.build(do_quantization=False)
if ret != 0:
print("Build failed!")
exit(ret)
print(model_path.replace('model.onnx',f'{target_platform}.rknn'))
ret = rknn.export_rknn(model_path.replace('model.onnx',f'{target_platform}.rknn'))
if ret != 0:
print('Export rknn model failed!')
exit(ret)
print('done')
del rknn
del RKNN
if not os.path.isfile(f'{model_path.replace("onnx","rknn")}'):
print(f'Dummy model not found at {model_path.replace("onnx","rknn")}, creating one')
with open(f'{model_path.replace("onnx","rknn")}', 'w'):
pass
if os.path.isdir(f'{args.model}/textual') and os.path.isdir(f'{args.model}/visual'): # is a clip model
print('Converting Clip model.')
ConvertModel(model_path=f'{args.model}/textual/model.onnx', target_platform=args.target_platform)
ConvertModel(model_path=f'{args.model}/visual/model.onnx', target_platform=args.target_platform)
elif os.path.isdir(f'{args.model}/detection') and os.path.isdir(f'{args.model}/recognition'): # is a facial model
print('Converting facial model.')
ConvertModel(f'{args.model}/detection/model.onnx', args.target_platform, [[[1, 3, 640, 640]]])
ConvertModel(f'{args.model}/recognition/model.onnx', args.target_platform, [[[1, 3, 112, 112]]])
else:
print('Unknown model.')

View file

@ -0,0 +1,13 @@
#!/bin/bash
python3 build_rknn.py "$1" "$2" > immich_to_rknn2.log
# if "No lowering found for" found in log file, return error status 1
if grep -q "No lowering found for" immich_to_rknn2.log; then
echo -e "\e[31mSome operations are not supported by RKNN, please check the log file for details.\e[0m"
exit 1
else
echo -e "\e[32mConversion completed successfully.\e[0m"
rm immich_to_rknn2.log
exit 0
fi

View file

@ -0,0 +1 @@
rknn-toolkit2==2.3.0

View file

@ -0,0 +1,83 @@
# This code is from leafqycc/rknn-multi-threaded
# Following Apache License 2.0
import os
from concurrent.futures import ThreadPoolExecutor
from queue import Queue
import numpy as np
from typing import Callable
from numpy.typing import NDArray
from app.config import log
supported_socs = ["rk3566", "rk3588"]
coremask_supported_socs = ["rk3576","rk3588"]
try:
from rknnlite.api import RKNNLite
with open("/proc/device-tree/compatible") as f:
device_compatible_str = f.read()
for soc in supported_socs:
if soc in device_compatible_str:
is_available = True
soc_name = soc
break
else:
is_available = False
soc_name = None
is_available = is_available and os.path.exists("/sys/kernel/debug/rknpu/load")
except (FileNotFoundError, ImportError):
log.debug("RKNN is not available")
is_available = False
soc_name = None
def init_rknn(rknnModel) -> Callable:
if not is_available:
raise RuntimeError("rknn is not available!")
rknn_lite = RKNNLite()
ret = rknn_lite.load_rknn(rknnModel)
if ret != 0:
raise RuntimeError("Load RKNN rknnModel failed")
if soc_name in coremask_supported_socs:
ret = rknn_lite.init_runtime(core_mask=RKNNLite.NPU_CORE_AUTO)
else:
ret = rknn_lite.init_runtime() # Please do not set this parameter on other platforms.
if ret != 0:
raise RuntimeError("Init runtime environment failed")
return rknn_lite
def init_rknns(rknnModel, tpes) -> list[Callable]:
rknn_list = []
for i in range(tpes):
rknn_list.append(init_rknn(rknnModel))
return rknn_list
class RknnPoolExecutor:
def __init__(self, rknnModel: str, tpes: int, func):
self.tpes = tpes
self.queue = Queue()
self.rknn_pool = init_rknns(rknnModel, tpes)
self.pool = ThreadPoolExecutor(max_workers=tpes)
self.func = func
self.num = 0
def put(self, inputs) -> None:
self.queue.put(self.pool.submit(self.func, self.rknn_pool[self.num % self.tpes], inputs))
self.num += 1
def get(self) -> list[list[NDArray[np.float32]], bool]:
if self.queue.empty():
return None
fut = self.queue.get()
return fut.result()
def release(self) -> None:
self.pool.shutdown()
for rknn_lite in self.rknn_pool:
rknn_lite.release()

21
web/package-lock.json generated
View file

@ -16,6 +16,8 @@
"@mdi/js": "^7.4.47",
"@photo-sphere-viewer/core": "^5.11.5",
"@photo-sphere-viewer/equirectangular-video-adapter": "^5.11.5",
"@photo-sphere-viewer/resolution-plugin": "^5.11.5",
"@photo-sphere-viewer/settings-plugin": "^5.11.5",
"@photo-sphere-viewer/video-plugin": "^5.11.5",
"@zoom-image/svelte": "^0.3.0",
"dom-to-image": "^2.6.0",
@ -1669,6 +1671,25 @@
"@photo-sphere-viewer/video-plugin": "5.11.5"
}
},
"node_modules/@photo-sphere-viewer/resolution-plugin": {
"version": "5.11.5",
"resolved": "https://registry.npmjs.org/@photo-sphere-viewer/resolution-plugin/-/resolution-plugin-5.11.5.tgz",
"integrity": "sha512-Dbvp5bBtozD3IWt1Q0wORVaZBcB1bV9xUeoOS9A7F7b3EkQ2pkC5/jot/1AyM4wtU5wJ63NWHskQ1d7m6WWazQ==",
"license": "MIT",
"peerDependencies": {
"@photo-sphere-viewer/core": "5.11.5",
"@photo-sphere-viewer/settings-plugin": "5.11.5"
}
},
"node_modules/@photo-sphere-viewer/settings-plugin": {
"version": "5.11.5",
"resolved": "https://registry.npmjs.org/@photo-sphere-viewer/settings-plugin/-/settings-plugin-5.11.5.tgz",
"integrity": "sha512-ZgYaWjiBMhsoRH5ddW3h+v4J4LPmofsT7BBRq5UCssWw2Fsrvv7mFFRi4UbZ1qzeKmvNUOr8BaFQgX1ZLvUWfQ==",
"license": "MIT",
"peerDependencies": {
"@photo-sphere-viewer/core": "5.11.5"
}
},
"node_modules/@photo-sphere-viewer/video-plugin": {
"version": "5.11.5",
"resolved": "https://registry.npmjs.org/@photo-sphere-viewer/video-plugin/-/video-plugin-5.11.5.tgz",

View file

@ -72,6 +72,8 @@
"@mdi/js": "^7.4.47",
"@photo-sphere-viewer/core": "^5.11.5",
"@photo-sphere-viewer/equirectangular-video-adapter": "^5.11.5",
"@photo-sphere-viewer/resolution-plugin": "^5.11.5",
"@photo-sphere-viewer/settings-plugin": "^5.11.5",
"@photo-sphere-viewer/video-plugin": "^5.11.5",
"@zoom-image/svelte": "^0.3.0",
"dom-to-image": "^2.6.0",

View file

@ -24,7 +24,7 @@
{:then [data, { default: PhotoSphereViewer }]}
<PhotoSphereViewer
panorama={data}
originalImageUrl={isWebCompatibleImage(asset) ? getAssetOriginalUrl(asset.id) : undefined}
originalPanorama={isWebCompatibleImage(asset) ? getAssetOriginalUrl(asset.id) : undefined}
/>
{:catch}
{$t('errors.failed_to_load_asset')}

View file

@ -7,18 +7,21 @@
type AdapterConstructor,
type PluginConstructor,
} from '@photo-sphere-viewer/core';
import { SettingsPlugin } from '@photo-sphere-viewer/settings-plugin';
import { ResolutionPlugin } from '@photo-sphere-viewer/resolution-plugin';
import '@photo-sphere-viewer/core/index.css';
import '@photo-sphere-viewer/settings-plugin/index.css';
import { onDestroy, onMount } from 'svelte';
interface Props {
panorama: string | { source: string };
originalImageUrl?: string;
originalPanorama?: string | { source: string };
adapter?: AdapterConstructor | [AdapterConstructor, unknown];
plugins?: (PluginConstructor | [PluginConstructor, unknown])[];
navbar?: boolean;
}
let { panorama, originalImageUrl, adapter = EquirectangularAdapter, plugins = [], navbar = false }: Props = $props();
let { panorama, originalPanorama, adapter = EquirectangularAdapter, plugins = [], navbar = false }: Props = $props();
let container: HTMLDivElement | undefined = $state();
let viewer: Viewer;
@ -30,9 +33,33 @@
viewer = new Viewer({
adapter,
plugins,
plugins: [
SettingsPlugin,
[
ResolutionPlugin,
{
defaultResolution: $alwaysLoadOriginalFile && originalPanorama ? 'original' : 'default',
resolutions: [
{
id: 'default',
label: 'Default',
panorama,
},
...(originalPanorama
? [
{
id: 'original',
label: 'Original',
panorama: originalPanorama,
},
]
: []),
],
},
],
...plugins,
],
container,
panorama,
touchmoveTwoFingers: false,
mousewheelCtrlKey: false,
navbar,
@ -40,15 +67,14 @@
maxFov: 120,
fisheye: false,
});
const resolutionPlugin = viewer.getPlugin(ResolutionPlugin) as ResolutionPlugin;
if (originalImageUrl && !$alwaysLoadOriginalFile) {
if (originalPanorama && !$alwaysLoadOriginalFile) {
const zoomHandler = ({ zoomLevel }: events.ZoomUpdatedEvent) => {
// zoomLevel range: [0, 100]
if (Math.round(zoomLevel) >= 75) {
// Replace the preview with the original
viewer.setPanorama(originalImageUrl, { showLoader: false, speed: 150 }).catch(() => {
viewer.setPanorama(panorama, { showLoader: false, speed: 0 }).catch(() => {});
});
void resolutionPlugin.setResolution('original');
viewer.removeEventListener(events.ZoomUpdatedEvent.type, zoomHandler);
}
};

View file

@ -1,5 +1,5 @@
<script lang="ts">
import { getAssetOriginalUrl } from '$lib/utils';
import { getAssetPlaybackUrl, getAssetOriginalUrl } from '$lib/utils';
import { fade } from 'svelte/transition';
import LoadingSpinner from '../shared-components/loading-spinner.svelte';
import { t } from 'svelte-i18n';
@ -22,7 +22,13 @@
{#await modules}
<LoadingSpinner />
{:then [PhotoSphereViewer, adapter, videoPlugin]}
<PhotoSphereViewer panorama={{ source: getAssetOriginalUrl(assetId) }} plugins={[videoPlugin]} {adapter} navbar />
<PhotoSphereViewer
panorama={{ source: getAssetPlaybackUrl(assetId) }}
originalPanorama={{ source: getAssetOriginalUrl(assetId) }}
plugins={[videoPlugin]}
{adapter}
navbar
/>
{:catch}
{$t('errors.failed_to_load_asset')}
{/await}