feat(web): use wasm for justified layout calculation (#15524)

* working * use wrapper class * update import * simplify * it works without changing `optimizeDeps` * inline layout options * update gallery view * use es2022 * fix import * fix vitest * empty geometry * bump version * Update web/src/lib/stores/assets.store.ts Co-authored-by: Jason Rasmussen <jason@rasm.me> * fix: typo --------- Co-authored-by: Jason Rasmussen <jason@rasm.me> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2025-04-21 15:36:26 +02:00 · 2025-02-21 12:20:25 +03:00 · 2025-02-21 12:20:25 +03:00 · 3925445de8
commit 3925445de8
parent 52f21fb331
9 changed files with 98 additions and 110 deletions
--- a/web/package-lock.json
+++ b/web/package-lock.json
@ -10,6 +10,7 @@
      "license": "GNU Affero General Public License version 3",
      "dependencies": {
        "@formatjs/icu-messageformat-parser": "^2.9.8",
+        "@immich/justified-layout-wasm": "^0.1.2",
        "@immich/sdk": "file:../open-api/typescript-sdk",
        "@immich/ui": "^0.16.0",
        "@mapbox/mapbox-gl-rtl-text": "0.2.3",
@ -23,7 +24,6 @@
        "dom-to-image": "^2.6.0",
        "handlebars": "^4.7.8",
        "intl-messageformat": "^10.7.11",
-        "justified-layout": "^4.1.0",
        "lodash-es": "^4.17.21",
        "luxon": "^3.4.4",
        "socket.io-client": "~4.8.0",
@ -45,7 +45,6 @@
        "@testing-library/svelte": "^5.2.6",
        "@testing-library/user-event": "^14.5.2",
        "@types/dom-to-image": "^2.6.7",
-        "@types/justified-layout": "^4.1.4",
        "@types/lodash-es": "^4.17.12",
        "@types/luxon": "^3.4.2",
        "@typescript-eslint/eslint-plugin": "^8.20.0",
@ -71,6 +70,7 @@
        "tslib": "^2.6.2",
        "typescript": "^5.7.3",
        "vite": "^6.0.0",
+        "vite-plugin-wasm": "^3.4.1",
        "vitest": "^3.0.0"
      }
    },
@ -1339,6 +1339,12 @@
        "url": "https://opencollective.com/libvips"
      }
    },
+    "node_modules/@immich/justified-layout-wasm": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/@immich/justified-layout-wasm/-/justified-layout-wasm-0.1.2.tgz",
+      "integrity": "sha512-6AmzYJhLedzIXSkhO/0tfBbHAeUeLmG1c4yTzJmtuSGyn7JAzVCFp0dp4T8Wh1tfIDx0Y0pAYB9tm2xlJHdEPA==",
+      "license": "AGPL-3"
+    },
    "node_modules/@immich/sdk": {
      "resolved": "../open-api/typescript-sdk",
      "link": true
@ -2413,12 +2419,6 @@
      "dev": true,
      "license": "MIT"
    },
-    "node_modules/@types/justified-layout": {
-      "version": "4.1.4",
-      "resolved": "https://registry.npmjs.org/@types/justified-layout/-/justified-layout-4.1.4.tgz",
-      "integrity": "sha512-q2ybP0u0NVj87oMnGZOGxY2iUN8ddr48zPOBHBdbOLpsMTA/keGj+93ou+OMCnJk0xewzlNIaVEkxM6VBD3E2w==",
-      "dev": true
-    },
    "node_modules/@types/leaflet": {
      "version": "1.9.8",
      "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.8.tgz",
@ -5382,11 +5382,6 @@
      "resolved": "https://registry.npmjs.org/just-flush/-/just-flush-2.3.0.tgz",
      "integrity": "sha512-fBuxQ1gJ61BurmhwKS5LYTzhkbrT5j/2U7ax+UbLm9aRvCTh2h6AfzLteOckE4KKomqOf0Y3zIG3Xu57sRsKUg=="
    },
-    "node_modules/justified-layout": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/justified-layout/-/justified-layout-4.1.0.tgz",
-      "integrity": "sha512-M5FimNMXgiOYerVRGsXZ2YK9YNCaTtwtYp7Hb2308U1Q9TXXHx5G0p08mcVR5O53qf8bWY4NJcPBxE6zuayXSg=="
-    },
    "node_modules/kdbush": {
      "version": "4.0.2",
      "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz",
@ -8625,6 +8620,16 @@
        }
      }
    },
+    "node_modules/vite-plugin-wasm": {
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/vite-plugin-wasm/-/vite-plugin-wasm-3.4.1.tgz",
+      "integrity": "sha512-ja3nSo2UCkVeitltJGkS3pfQHAanHv/DqGatdI39ja6McgABlpsZ5hVgl6wuR8Qx5etY3T5qgDQhOWzc5RReZA==",
+      "dev": true,
+      "license": "MIT",
+      "peerDependencies": {
+        "vite": "^2 || ^3 || ^4 || ^5 || ^6"
+      }
+    },
    "node_modules/vitefu": {
      "version": "1.0.5",
      "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.0.5.tgz",
--- a/web/package.json
+++ b/web/package.json
@ -35,7 +35,6 @@
    "@testing-library/svelte": "^5.2.6",
    "@testing-library/user-event": "^14.5.2",
    "@types/dom-to-image": "^2.6.7",
-    "@types/justified-layout": "^4.1.4",
    "@types/lodash-es": "^4.17.12",
    "@types/luxon": "^3.4.2",
    "@typescript-eslint/eslint-plugin": "^8.20.0",
@ -61,11 +60,13 @@
    "tslib": "^2.6.2",
    "typescript": "^5.7.3",
    "vite": "^6.0.0",
+    "vite-plugin-wasm": "^3.4.1",
    "vitest": "^3.0.0"
  },
  "type": "module",
  "dependencies": {
    "@formatjs/icu-messageformat-parser": "^2.9.8",
+    "@immich/justified-layout-wasm": "^0.1.2",
    "@immich/sdk": "file:../open-api/typescript-sdk",
    "@immich/ui": "^0.16.0",
    "@mapbox/mapbox-gl-rtl-text": "0.2.3",
@ -79,7 +80,6 @@
    "dom-to-image": "^2.6.0",
    "handlebars": "^4.7.8",
    "intl-messageformat": "^10.7.11",
-    "justified-layout": "^4.1.0",
    "lodash-es": "^4.17.21",
    "luxon": "^3.4.4",
    "socket.io-client": "~4.8.0",
--- a/web/src/lib/components/photos-page/asset-date-group.svelte
+++ b/web/src/lib/components/photos-page/asset-date-group.svelte
@ -97,6 +97,7 @@
  {#each dateGroups as dateGroup, groupIndex (dateGroup.date)}
    {@const display =
      dateGroup.intersecting || !!dateGroup.assets.some((asset) => asset.id === $assetStore.pendingScrollAssetId)}
+    {@const geometry = dateGroup.geometry}

    <div
      id="date-group"
@ -118,7 +119,7 @@
      data-display={display}
      data-date-group={dateGroup.date}
      style:height={dateGroup.height + 'px'}
-      style:width={dateGroup.geometry.containerWidth + 'px'}
+      style:width={geometry.containerWidth + 'px'}
      style:overflow={'clip'}
    >
      {#if !display}
@ -149,7 +150,7 @@
          <!-- Date group title -->
          <div
            class="flex z-[100] sticky top-[-1px] pt-[calc(1.75rem+1px)] pb-5 h-6 place-items-center text-xs font-medium text-immich-fg bg-immich-bg dark:bg-immich-dark-bg dark:text-immich-dark-fg md:text-sm"
-            style:width={dateGroup.geometry.containerWidth + 'px'}
+            style:width={geometry.containerWidth + 'px'}
          >
            {#if !singleSelect && ((hoveredDateGroup == dateGroup.groupTitle && isMouseOverGroup) || assetInteraction.selectedGroup.has(dateGroup.groupTitle))}
              <div
@ -174,12 +175,17 @@
          <!-- Image grid -->
          <div
            class="relative overflow-clip"
-            style:height={dateGroup.geometry.containerHeight + 'px'}
-            style:width={dateGroup.geometry.containerWidth + 'px'}
+            style:height={geometry.containerHeight + 'px'}
+            style:width={geometry.containerWidth + 'px'}
          >
-            {#each dateGroup.assets as asset, index (asset.id)}
-              {@const box = dateGroup.geometry.boxes[index]}
+            {#each dateGroup.assets as asset, i (asset.id)}
              {@const isSmallGroup = dateGroup.assets.length <= SMALL_GROUP_THRESHOLD}
+              <!-- getting these together here in this order is very cache-efficient -->
+              {@const top = geometry.getTop(i)}
+              {@const left = geometry.getLeft(i)}
+              {@const width = geometry.getWidth(i)}
+              {@const height = geometry.getHeight(i)}
+
              <!-- update ASSET_GRID_PADDING-->
              <div
                use:intersectionObserver={{
@ -191,10 +197,10 @@
                }}
                data-asset-id={asset.id}
                class="absolute"
-                style:width={box.width + 'px'}
-                style:height={box.height + 'px'}
-                style:top={box.top + 'px'}
-                style:left={box.left + 'px'}
+                style:top={top + 'px'}
+                style:left={left + 'px'}
+                style:width={width + 'px'}
+                style:height={height + 'px'}
              >
                <Thumbnail
                  {dateGroup}
@ -216,8 +222,8 @@
                  selected={assetInteraction.selectedAssets.has(asset) || $assetStore.albumAssets.has(asset.id)}
                  selectionCandidate={assetInteraction.assetSelectionCandidates.has(asset)}
                  disabled={$assetStore.albumAssets.has(asset.id)}
-                  thumbnailWidth={box.width}
-                  thumbnailHeight={box.height}
+                  thumbnailWidth={width}
+                  thumbnailHeight={height}
                  eagerThumbhash={isSmallGroup}
                />
              </div>
--- a/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte
+++ b/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte
@ -8,13 +8,11 @@
  import type { Viewport } from '$lib/stores/assets.store';
  import { showDeleteModal } from '$lib/stores/preferences.store';
  import { deleteAssets } from '$lib/utils/actions';
-  import { archiveAssets, cancelMultiselect, getAssetRatio } from '$lib/utils/asset-utils';
+  import { archiveAssets, cancelMultiselect, getJustifiedLayoutFromAssets } from '$lib/utils/asset-utils';
  import { featureFlags } from '$lib/stores/server-config.store';
  import { handleError } from '$lib/utils/handle-error';
  import { navigate } from '$lib/utils/navigation';
-  import { calculateWidth } from '$lib/utils/timeline-util';
  import { type AssetResponseDto } from '@immich/sdk';
-  import justifiedLayout from 'justified-layout';
  import { t } from 'svelte-i18n';
  import AssetViewer from '../../asset-viewer/asset-viewer.svelte';
  import ShowShortcuts from '../show-shortcuts.svelte';
@ -310,23 +308,12 @@
  let idsSelectedAssets = $derived(assetInteraction.selectedAssetsArray.map(({ id }) => id));

  let geometry = $derived(
-    (() => {
-      const justifiedLayoutResult = justifiedLayout(
-        assets.map((asset) => getAssetRatio(asset)),
-        {
-          boxSpacing: 2,
-          containerWidth: Math.floor(viewport.width),
-          containerPadding: 0,
-          targetRowHeightTolerance: 0.15,
-          targetRowHeight: 235,
-        },
-      );
-
-      return {
-        ...justifiedLayoutResult,
-        containerWidth: calculateWidth(justifiedLayoutResult.boxes),
-      };
-    })(),
+    getJustifiedLayoutFromAssets(assets, {
+      spacing: 2,
+      rowWidth: Math.floor(viewport.width),
+      heightTolerance: 0.15,
+      rowHeight: 235,
+    }),
  );

  $effect(() => {
@ -364,11 +351,15 @@

 {#if assets.length > 0}
  <div class="relative" style="height: {geometry.containerHeight}px;width: {geometry.containerWidth}px ">
-    {#each assets as asset, i (i)}
+    {#each assets as asset, i}
+      {@const top = geometry.getTop(i)}
+      {@const left = geometry.getLeft(i)}
+      {@const width = geometry.getWidth(i)}
+      {@const height = geometry.getHeight(i)}
+
      <div
        class="absolute"
-        style="width: {geometry.boxes[i].width}px; height: {geometry.boxes[i].height}px; top: {geometry.boxes[i]
-          .top}px; left: {geometry.boxes[i].left}px"
+        style="width: {width}px; height: {height}px; top: {top}px; left: {left}px"
        title={showAssetName ? asset.originalFileName : ''}
      >
        <Thumbnail
@ -387,8 +378,8 @@
          {asset}
          selected={assetInteraction.selectedAssets.has(asset)}
          selectionCandidate={assetInteraction.assetSelectionCandidates.has(asset)}
-          thumbnailWidth={geometry.boxes[i].width}
-          thumbnailHeight={geometry.boxes[i].height}
+          thumbnailWidth={width}
+          thumbnailHeight={height}
        />
        {#if showAssetName}
          <div
--- a/web/src/lib/stores/assets.store.ts
+++ b/web/src/lib/stores/assets.store.ts
@ -1,12 +1,11 @@
 import { locale } from '$lib/stores/preferences.store';
 import { getKey } from '$lib/utils';
 import { AssetGridTaskManager } from '$lib/utils/asset-store-task-manager';
-import { getAssetRatio } from '$lib/utils/asset-utils';
+import { getJustifiedLayoutFromAssets } from '$lib/utils/asset-utils';
 import { generateId } from '$lib/utils/generate-id';
 import type { AssetGridRouteSearchParams } from '$lib/utils/navigation';
-import { calculateWidth, fromLocalDateTime, splitBucketIntoDateGroups, type DateGroup } from '$lib/utils/timeline-util';
+import { fromLocalDateTime, splitBucketIntoDateGroups, type DateGroup } from '$lib/utils/timeline-util';
 import { TimeBucketSize, getAssetInfo, getTimeBucket, getTimeBuckets, type AssetResponseDto } from '@immich/sdk';
-import createJustifiedLayout from 'justified-layout';
 import { throttle } from 'lodash-es';
 import { DateTime } from 'luxon';
 import { t } from 'svelte-i18n';
@ -16,13 +15,6 @@ import { websocketEvents } from './websocket';
 type AssetApiGetTimeBucketsRequest = Parameters<typeof getTimeBuckets>[0];
 export type AssetStoreOptions = Omit<AssetApiGetTimeBucketsRequest, 'size'>;

-const LAYOUT_OPTIONS = {
-  boxSpacing: 2,
-  containerPadding: 0,
-  targetRowHeightTolerance: 0.15,
-  targetRowHeight: 235,
-};
-
 export interface Viewport {
  width: number;
  height: number;
@ -470,32 +462,30 @@ export class AssetStore {
        assetGroup.heightActual = false;
      }
    }
+
+    const viewportWidth = this.viewport.width;
    if (!bucket.isBucketHeightActual) {
      const unwrappedWidth = (3 / 2) * bucket.bucketCount * THUMBNAIL_HEIGHT * (7 / 10);
-      const rows = Math.ceil(unwrappedWidth / this.viewport.width);
+      const rows = Math.ceil(unwrappedWidth / viewportWidth);
      const height = 51 + rows * THUMBNAIL_HEIGHT;
      bucket.bucketHeight = height;
    }

+    const layoutOptions = {
+      spacing: 2,
+      heightTolerance: 0.15,
+      rowHeight: 235,
+      rowWidth: Math.floor(viewportWidth),
+    };
    for (const assetGroup of bucket.dateGroups) {
      if (!assetGroup.heightActual) {
        const unwrappedWidth = (3 / 2) * assetGroup.assets.length * THUMBNAIL_HEIGHT * (7 / 10);
-        const rows = Math.ceil(unwrappedWidth / this.viewport.width);
+        const rows = Math.ceil(unwrappedWidth / viewportWidth);
        const height = rows * THUMBNAIL_HEIGHT;
        assetGroup.height = height;
      }

-      const layoutResult = createJustifiedLayout(
-        assetGroup.assets.map((g) => getAssetRatio(g)),
-        {
-          ...LAYOUT_OPTIONS,
-          containerWidth: Math.floor(this.viewport.width),
-        },
-      );
-      assetGroup.geometry = {
-        ...layoutResult,
-        containerWidth: calculateWidth(layoutResult.boxes),
-      };
+      assetGroup.geometry = getJustifiedLayoutFromAssets(assetGroup.assets, layoutOptions);
    }
  }

--- a/web/src/lib/utils/asset-utils.ts
+++ b/web/src/lib/utils/asset-utils.ts
@ -12,6 +12,7 @@ import { downloadRequest, getKey, withError } from '$lib/utils';
 import { createAlbum } from '$lib/utils/album-utils';
 import { getByteUnitString } from '$lib/utils/byte-units';
 import { getFormatter } from '$lib/utils/i18n';
+import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm';
 import {
  addAssetsToAlbum as addAssets,
  createStack,
@ -587,3 +588,13 @@ export const copyImageToClipboard = async (source: HTMLImageElement | string) =>
  const blob = source instanceof HTMLImageElement ? await imgToBlob(source) : await urlToBlob(source);
  await navigator.clipboard.write([new ClipboardItem({ [blob.type]: blob })]);
 };
+
+export function getJustifiedLayoutFromAssets(assets: AssetResponseDto[], options: LayoutOptions) {
+  const aspectRatios = new Float32Array(assets.length);
+  // eslint-disable-next-line unicorn/no-for-loop
+  for (let i = 0; i < assets.length; i++) {
+    const { width, height } = getAssetRatio(assets[i]);
+    aspectRatios[i] = width / height;
+  }
+  return new JustifiedLayout(aspectRatios, options);
+}
--- a/web/src/lib/utils/timeline-util.ts
+++ b/web/src/lib/utils/timeline-util.ts
@ -1,7 +1,7 @@
 import type { AssetBucket } from '$lib/stores/assets.store';
 import { locale } from '$lib/stores/preferences.store';
+import { JustifiedLayout } from '@immich/justified-layout-wasm';
 import type { AssetResponseDto } from '@immich/sdk';
-import type createJustifiedLayout from 'justified-layout';
 import { groupBy, memoize, sortBy } from 'lodash-es';
 import { DateTime } from 'luxon';
 import { get } from 'svelte/store';
@ -13,7 +13,7 @@ export type DateGroup = {
  height: number;
  heightActual: boolean;
  intersecting: boolean;
-  geometry: Geometry;
+  geometry: JustifiedLayout;
  bucket: AssetBucket;
 };
 export type ScrubberListener = (
@ -80,18 +80,12 @@ export function formatGroupTitle(_date: DateTime): string {
  return date.toLocaleString(groupDateFormat);
 }

-type Geometry = ReturnType<typeof createJustifiedLayout> & {
-  containerWidth: number;
-};
-
-function emptyGeometry() {
-  return {
-    containerWidth: 0,
-    containerHeight: 0,
-    widowCount: 0,
-    boxes: [],
-  };
-}
+const emptyGeometry = new JustifiedLayout(Float32Array.from([]), {
+  rowHeight: 1,
+  heightTolerance: 0,
+  rowWidth: 1,
+  spacing: 0,
+});

 const formatDateGroupTitle = memoize(formatGroupTitle);

@ -100,6 +94,7 @@ export function splitBucketIntoDateGroups(bucket: AssetBucket, locale: string |
    fromLocalDateTime(asset.localDateTime).toLocaleString(groupDateFormat, { locale }),
  );
  const sorted = sortBy(grouped, (group) => bucket.assets.indexOf(group[0]));
+
  return sorted.map((group) => {
    const date = fromLocalDateTime(group[0].localDateTime).startOf('day');
    return {
@ -109,31 +104,12 @@ export function splitBucketIntoDateGroups(bucket: AssetBucket, locale: string |
      height: 0,
      heightActual: false,
      intersecting: false,
-      geometry: emptyGeometry(),
+      geometry: emptyGeometry,
      bucket,
    };
  });
 }

-export type LayoutBox = {
-  aspectRatio: number;
-  top: number;
-  width: number;
-  height: number;
-  left: number;
-  forcedAspectRatio?: boolean;
-};
-
-export function calculateWidth(boxes: LayoutBox[]): number {
-  let width = 0;
-  for (const box of boxes) {
-    if (box.top < 100) {
-      width = box.left + box.width;
-    }
-  }
-  return width;
-}
-
 export function findTotalOffset(element: HTMLElement, stop: HTMLElement) {
  let offset = 0;
  while (element.offsetParent && element !== stop) {
--- a/web/tsconfig.json
+++ b/web/tsconfig.json
@ -4,7 +4,7 @@
    "checkJs": true,
    "esModuleInterop": true,
    "forceConsistentCasingInFileNames": true,
-    "module": "es2020",
+    "module": "es2022",
    "moduleResolution": "bundler",
    "resolveJsonModule": true,
    "skipLibCheck": true,
--- a/web/vite.config.js
+++ b/web/vite.config.js
@ -4,6 +4,7 @@ import { svelteTesting } from '@testing-library/svelte/vite';
 import path from 'node:path';
 import { visualizer } from 'rollup-plugin-visualizer';
 import { defineConfig } from 'vite';
+import wasm from 'vite-plugin-wasm';

 const upstream = {
  target: process.env.IMMICH_SERVER_URL || 'http://immich-server:2283/',
@ -14,6 +15,9 @@ const upstream = {
 };

 export default defineConfig({
+  build: {
+    target: 'es2022',
+  },
  resolve: {
    alias: {
      'xmlhttprequest-ssl': './node_modules/engine.io-client/lib/xmlhttprequest.js',
@ -40,6 +44,7 @@ export default defineConfig({
      : undefined,
    enhancedImages(),
    svelteTesting(),
+    wasm(),
  ],
  optimizeDeps: {
    entries: ['src/**/*.{svelte,ts,html}'],
@ -52,5 +57,9 @@ export default defineConfig({
    sequence: {
      hooks: 'list',
    },
+    deps: {
+      // workaround for https://github.com/vitest-dev/vitest/issues/2150
+      inline: ['@immich/justified-layout-wasm'],
+    },
  },
 });