diff --git a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.spec.ts b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.spec.ts
index c82d9f9659..bf6ea170fc 100644
--- a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.spec.ts
+++ b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.spec.ts
@@ -9,7 +9,6 @@ describe('AssetViewerNavBar component', () => {
   const additionalProps = {
     showCopyButton: false,
     showZoomButton: false,
-    showDetailButton: false,
     showDownloadButton: false,
     showMotionPlayButton: false,
     showShareButton: false,
diff --git a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte
index a57a7faef8..988433bbfb 100644
--- a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte
+++ b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte
@@ -1,4 +1,5 @@
 <script lang="ts">
+  import { access } from '$lib/stores/access.store';
   import type { OnAction } from '$lib/components/asset-viewer/actions/action';
   import AddToAlbumAction from '$lib/components/asset-viewer/actions/add-to-album-action.svelte';
   import ArchiveAction from '$lib/components/asset-viewer/actions/archive-action.svelte';
@@ -17,7 +18,7 @@
   import MenuOption from '$lib/components/shared-components/context-menu/menu-option.svelte';
   import { user } from '$lib/stores/user.store';
   import { photoZoomState } from '$lib/stores/zoom-image.store';
-  import { getAssetJobName, getSharedLink } from '$lib/utils';
+  import { getAssetJobName } from '$lib/utils';
   import { openFileUploadDialog } from '$lib/utils/file-uploader';
   import {
     AssetJobName,
@@ -44,7 +45,6 @@
   export let asset: AssetResponseDto;
   export let album: AlbumResponseDto | null = null;
   export let stack: StackResponseDto | null = null;
-  export let showDetailButton: boolean;
   export let showSlideshow = false;
   export let onZoomImage: () => void;
   export let onCopyImage: () => void;
@@ -55,10 +55,7 @@
   // export let showEditorHandler: () => void;
   export let onClose: () => void;
 
-  const sharedLink = getSharedLink();
-
   $: isOwner = $user && asset.ownerId === $user?.id;
-  $: showDownloadButton = sharedLink ? sharedLink.allowDownload : !asset.isOffline;
   // $: showEditorButton =
   //   isOwner &&
   //   asset.type === AssetTypeEnum.Image &&
@@ -80,7 +77,7 @@
     class="flex w-[calc(100%-3rem)] justify-end gap-2 overflow-hidden text-white"
     data-testid="asset-viewer-navbar-actions"
   >
-    {#if !asset.isTrashed && $user}
+    {#if $access({ asset, access: 'asset.share' })}
       <ShareAction {asset} />
     {/if}
     {#if asset.isOffline}
@@ -102,15 +99,16 @@
       <CircleIconButton color="opaque" icon={mdiContentCopy} title={$t('copy_image')} on:click={onCopyImage} />
     {/if}
 
-    {#if !isOwner && showDownloadButton}
+    <!-- owner already has download in the overflow menu -->
+    {#if !isOwner && !asset.isOffline && $access({ asset, access: 'asset.download' })}
       <DownloadAction {asset} />
     {/if}
 
-    {#if showDetailButton}
+    {#if asset.hasMetadata}
       <ShowDetailAction {onShowDetail} />
     {/if}
 
-    {#if isOwner}
+    {#if $access({ asset, access: 'asset.favorite' })}
       <FavoriteAction {asset} {onAction} />
     {/if}
     <!-- {#if showEditorButton}
@@ -124,16 +122,18 @@
     {/if} -->
 
     {#if isOwner}
-      <DeleteAction {asset} {onAction} />
+      {#if !asset.isTrashed && $access({ asset, access: 'asset.delete' })}
+        <DeleteAction {asset} {onAction} />
+      {/if}
 
       <ButtonContextMenu direction="left" align="top-right" color="opaque" title={$t('more')} icon={mdiDotsVertical}>
         {#if showSlideshow}
           <MenuOption icon={mdiPresentationPlay} text={$t('slideshow')} onClick={onPlaySlideshow} />
         {/if}
-        {#if showDownloadButton}
+        {#if !asset.isOffline && $access({ asset, access: 'asset.download' })}
           <DownloadAction {asset} menuItem />
         {/if}
-        {#if asset.isTrashed}
+        {#if asset.isTrashed && $access({ asset, access: 'asset.delete' })}
           <RestoreAction {asset} {onAction} />
         {:else}
           <AddToAlbumAction {asset} {onAction} />
@@ -141,10 +141,11 @@
         {/if}
 
         {#if isOwner}
-          {#if stack}
+          {#if stack && $access({ asset, access: 'asset.stack' })}
             <UnstackAction {stack} {onAction} />
           {/if}
-          {#if album}
+
+          {#if album && $access({ album, access: 'album.update' })}
             <SetAlbumCoverAction {asset} {album} />
           {/if}
           {#if asset.type === AssetTypeEnum.Image}
diff --git a/web/src/lib/components/asset-viewer/asset-viewer.svelte b/web/src/lib/components/asset-viewer/asset-viewer.svelte
index 4e98546069..60ede24f93 100644
--- a/web/src/lib/components/asset-viewer/asset-viewer.svelte
+++ b/web/src/lib/components/asset-viewer/asset-viewer.svelte
@@ -77,7 +77,6 @@
   let appearsInAlbums: AlbumResponseDto[] = [];
   let shouldPlayMotionPhoto = false;
   let sharedLink = getSharedLink();
-  let enableDetailPanel = asset.hasMetadata;
   let slideshowStateUnsubscribe: () => void;
   let shuffleSlideshowUnsubscribe: () => void;
   let previewStackedAsset: AssetResponseDto | undefined;
@@ -415,7 +414,6 @@
         {asset}
         {album}
         {stack}
-        showDetailButton={enableDetailPanel}
         showSlideshow={!!assetStore}
         onZoomImage={zoomToggle}
         onCopyImage={copyImage}
@@ -531,7 +529,7 @@
     </div>
   {/if}
 
-  {#if enableDetailPanel && $slideshowState === SlideshowState.None && $isShowDetail && !isShowEditor}
+  {#if asset.hasMetadata && $slideshowState === SlideshowState.None && $isShowDetail && !isShowEditor}
     <div
       transition:fly={{ duration: 150 }}
       id="detail-panel"
diff --git a/web/src/lib/stores/access.store.spec.ts b/web/src/lib/stores/access.store.spec.ts
new file mode 100644
index 0000000000..fa2ebc4cc5
--- /dev/null
+++ b/web/src/lib/stores/access.store.spec.ts
@@ -0,0 +1,112 @@
+import { hasAccess } from '$lib/stores/access.store';
+import { albumFactory } from '@test-data/factories/album-factory';
+import { assetFactory } from '@test-data/factories/asset-factory';
+import { sharedLinkFactory } from '@test-data/factories/shared-link-factory';
+import { userAdminFactory, userFactory } from '@test-data/factories/user-factory';
+import { describe, it } from 'vitest';
+
+const user = userAdminFactory.build({ id: 'user-id' });
+
+describe('AccessStore', () => {
+  describe('hasAccess', () => {
+    describe('asset.share', () => {
+      const access = 'asset.share';
+
+      it('should return true for owned assets', () => {
+        const asset = assetFactory.build({ ownerId: user.id });
+        expect(hasAccess({ access, asset }, { user })).toBe(true);
+      });
+
+      it('should return false for not owned assets', () => {
+        const asset = assetFactory.build({ isOffline: false });
+        expect(hasAccess({ access, asset }, { user })).toBe(false);
+      });
+
+      it('should return false for trashed assets', () => {
+        const asset = assetFactory.build({ ownerId: user.id, isTrashed: true });
+        expect(hasAccess({ access, asset }, { user })).toBe(true);
+      });
+    });
+
+    describe('asset.download', () => {
+      const access = 'asset.download';
+
+      it('should return true for owned assets', () => {
+        const asset = assetFactory.build({ ownerId: user.id, isOffline: false });
+        expect(hasAccess({ access, asset }, { user })).toBe(true);
+      });
+
+      it('should return false for not owned assets', () => {
+        const asset = assetFactory.build({ isOffline: false });
+        expect(hasAccess({ access, asset }, { user })).toBe(false);
+      });
+
+      it('should return true for partner sharing', () => {
+        const asset = assetFactory.build({ ownerId: user.id, isOffline: false });
+        const partner = userFactory.build();
+        expect(hasAccess({ access, asset }, { user, partners: [partner] })).toBe(true);
+      });
+
+      it('should return false for shared links', () => {
+        const asset = assetFactory.build({ isOffline: false });
+        const sharedLink = sharedLinkFactory.build({ allowDownload: false });
+        expect(hasAccess({ access, asset }, { sharedLink })).toBe(false);
+      });
+
+      it('should return true for shared links with download enabled', () => {
+        const asset = assetFactory.build({ isOffline: false });
+        const sharedLink = sharedLinkFactory.build({ allowDownload: true });
+        expect(hasAccess({ access, asset }, { sharedLink })).toBe(true);
+      });
+    });
+
+    describe('asset.unstack', () => {
+      const access = 'asset.unstack';
+
+      it('should return true for owned assets', () => {
+        const asset = assetFactory.build({ ownerId: user.id });
+        expect(hasAccess({ access, asset }, { user })).toBe(true);
+      });
+
+      it('should return false for non owned assets', () => {
+        const asset = assetFactory.build();
+        expect(hasAccess({ access, asset }, { user })).toBe(false);
+      });
+
+      it('should return false for partner sharing', () => {
+        const partner = userFactory.build({ id: 'partner-id' });
+        const asset = assetFactory.build({ ownerId: partner.id });
+        expect(hasAccess({ access, asset }, { user, partners: [partner] })).toBe(false);
+      });
+    });
+
+    describe('asset.favorite', () => {
+      const access = 'asset.favorite';
+
+      it('should return true for owned assets', () => {
+        const asset = assetFactory.build({ ownerId: user.id });
+        expect(hasAccess({ access, asset }, { user })).toBe(true);
+      });
+
+      it('should return false for non owned assets', () => {
+        const asset = assetFactory.build();
+        expect(hasAccess({ access, asset }, { user })).toBe(false);
+      });
+
+      it('should return false for partner sharing', () => {
+        const partner = userFactory.build({ id: 'partner-id' });
+        const asset = assetFactory.build({ ownerId: partner.id });
+        expect(hasAccess({ access, asset }, { user, partners: [partner] })).toBe(false);
+      });
+    });
+
+    describe('album.update', () => {
+      const access = 'album.update';
+
+      it('should return true for owned albums', () => {
+        const album = albumFactory.build({ ownerId: user.id });
+        expect(hasAccess({ access, album }, { user })).toBe(true);
+      });
+    });
+  });
+});
diff --git a/web/src/lib/stores/access.store.ts b/web/src/lib/stores/access.store.ts
new file mode 100644
index 0000000000..726e3fe230
--- /dev/null
+++ b/web/src/lib/stores/access.store.ts
@@ -0,0 +1,104 @@
+import { user as userStore } from '$lib/stores/user.store';
+import { getSharedLink } from '$lib/utils';
+import {
+  getPartners,
+  PartnerDirection,
+  Permission,
+  type AlbumResponseDto,
+  type AssetResponseDto,
+  type PartnerResponseDto,
+  type SharedLinkResponseDto,
+  type UserAdminResponseDto,
+} from '@immich/sdk';
+import { derived, writable } from 'svelte/store';
+
+type AssetPermissions =
+  | 'asset.delete'
+  | 'asset.share'
+  | 'asset.download'
+  | 'asset.upload'
+  | 'asset.favorite'
+  | 'asset.stack'
+  | 'asset.unstack';
+
+type AlbumPermissions = 'album.update';
+
+type AssetAccessRequest = { asset: AssetResponseDto; access: AssetPermissions };
+type AlbumAccessRequest = { album: AlbumResponseDto; access: AlbumPermissions };
+type AccessRequest = AssetAccessRequest | AlbumAccessRequest;
+
+type AccessData = {
+  user?: UserAdminResponseDto;
+  partners?: PartnerResponseDto[];
+  sharedLink?: SharedLinkResponseDto;
+};
+
+const isAssetRequest = (request: AccessRequest): request is AssetAccessRequest => 'asset' in request;
+const isAlbumRequest = (request: AccessRequest): request is AlbumAccessRequest => 'album' in request;
+
+export const hasAccess = (request: AccessRequest, data: AccessData) => {
+  return hasUserAccess(request, data) || hasSharedLinkAccess(request, data);
+};
+
+const hasUserAccess = (request: AccessRequest, { user, partners }: AccessData) => {
+  if (!user) {
+    return false;
+  }
+
+  const partnerIds = new Set((partners || []).map((partner) => partner.id));
+
+  if (isAssetRequest(request)) {
+    const { asset, access } = request;
+    if (asset.ownerId === user.id) {
+      return true;
+    }
+
+    if ((['asset.download'] as AssetPermissions[]).includes(access) && partnerIds.has(asset.ownerId)) {
+      return true;
+    }
+  }
+
+  if (isAlbumRequest(request)) {
+    const { album } = request;
+    if (album.ownerId === user.id) {
+      return true;
+    }
+  }
+
+  return false;
+};
+
+const hasSharedLinkAccess = (request: AccessRequest, { sharedLink }: AccessData) => {
+  if (!sharedLink) {
+    return false;
+  }
+
+  switch (request.access) {
+    case Permission.AssetUpload: {
+      return sharedLink.allowUpload;
+    }
+
+    case Permission.AssetDownload: {
+      return sharedLink.allowDownload;
+    }
+
+    default: {
+      return false;
+    }
+  }
+};
+
+const partnerStore = writable<PartnerResponseDto[]>([]);
+export const access = derived([userStore, partnerStore], ([user, partners]) => {
+  const data = { user, partners, sharedLink: getSharedLink() };
+  return (request: AccessRequest) => hasAccess(request, data);
+});
+
+const init = async () => {
+  const partners = await getPartners({ direction: PartnerDirection.SharedWith });
+  partnerStore.set(partners);
+};
+
+export const accessManager = {
+  init,
+};
diff --git a/web/src/lib/utils/auth.ts b/web/src/lib/utils/auth.ts
index d37f1bb960..2f64909839 100644
--- a/web/src/lib/utils/auth.ts
+++ b/web/src/lib/utils/auth.ts
@@ -1,4 +1,5 @@
 import { browser } from '$app/environment';
+import { accessManager } from '$lib/stores/access.store';
 import { purchaseStore } from '$lib/stores/purchase.store';
 import { serverInfo } from '$lib/stores/server-info.store';
 import { preferences as preferences$, user as user$ } from '$lib/stores/user.store';
@@ -24,6 +25,9 @@ export const loadUser = async () => {
       user$.set(user);
       preferences$.set(preferences);
 
+      // TODO invert (emit an event that accessManager listens to)
+      await accessManager.init();
+
       // Check for license status
       if (serverInfo.licensed || user.license?.activatedAt) {
         purchaseStore.setPurchaseStatus(true);