From 138e2d915869d14365bb69c3d65366ba314b8f03 Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:49:57 -0400 Subject: [PATCH] feat(web): hls player (#28312) * update e2e * hls player * fix transcoding restart on explicit quality selection * move level filtering to manager * move init to manager declaration * refactor commit on release * these lints... * fix seek sometimes being ignored * fix panic downswitch --- e2e/src/specs/server/api/server.e2e-spec.ts | 1 + i18n/en.json | 1 + .../lib/model/server_features_dto.dart | 11 +- open-api/immich-openapi-specs.json | 5 + packages/sdk/src/fetch-client.ts | 2 + pnpm-lock.yaml | 30 ++ server/src/dtos/server.dto.ts | 1 + server/src/services/server.service.spec.ts | 1 + server/src/services/server.service.ts | 3 +- web/package.json | 2 + .../asset-viewer/VideoNativeViewer.svelte | 274 +++++++++++++++--- .../asset-viewer/immich-time-range.ts | 54 ++++ .../media-capabilities-manager.svelte.ts | 92 ++++++ web/src/lib/utils.ts | 8 + 14 files changed, 449 insertions(+), 36 deletions(-) create mode 100644 web/src/lib/components/asset-viewer/immich-time-range.ts create mode 100644 web/src/lib/managers/media-capabilities-manager.svelte.ts diff --git a/e2e/src/specs/server/api/server.e2e-spec.ts b/e2e/src/specs/server/api/server.e2e-spec.ts index f15d450213..9ab2f5d823 100644 --- a/e2e/src/specs/server/api/server.e2e-spec.ts +++ b/e2e/src/specs/server/api/server.e2e-spec.ts @@ -116,6 +116,7 @@ describe('/server', () => { oauthAutoLaunch: false, ocr: false, passwordLogin: true, + realtimeTranscoding: false, search: true, sidecar: true, trash: true, diff --git a/i18n/en.json b/i18n/en.json index 035a662545..90beb7077e 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -2460,6 +2460,7 @@ "video": "Video", "video_hover_setting": "Play video thumbnail on hover", "video_hover_setting_description": "Play video thumbnail when mouse is hovering over item. Even when disabled, playback can be started by hovering over the play icon.", + "video_quality": "Video quality", "videos": "Videos", "videos_count": "{count, plural, one {# Video} other {# Videos}}", "videos_only": "Videos only", diff --git a/mobile/openapi/lib/model/server_features_dto.dart b/mobile/openapi/lib/model/server_features_dto.dart index 79494b74eb..9b75ef2b32 100644 --- a/mobile/openapi/lib/model/server_features_dto.dart +++ b/mobile/openapi/lib/model/server_features_dto.dart @@ -23,6 +23,7 @@ class ServerFeaturesDto { required this.oauthAutoLaunch, required this.ocr, required this.passwordLogin, + required this.realtimeTranscoding, required this.reverseGeocoding, required this.search, required this.sidecar, @@ -60,6 +61,9 @@ class ServerFeaturesDto { /// Whether password login is enabled bool passwordLogin; + /// Whether real-time transcoding is enabled + bool realtimeTranscoding; + /// Whether reverse geocoding is enabled bool reverseGeocoding; @@ -87,6 +91,7 @@ class ServerFeaturesDto { other.oauthAutoLaunch == oauthAutoLaunch && other.ocr == ocr && other.passwordLogin == passwordLogin && + other.realtimeTranscoding == realtimeTranscoding && other.reverseGeocoding == reverseGeocoding && other.search == search && other.sidecar == sidecar && @@ -106,6 +111,7 @@ class ServerFeaturesDto { (oauthAutoLaunch.hashCode) + (ocr.hashCode) + (passwordLogin.hashCode) + + (realtimeTranscoding.hashCode) + (reverseGeocoding.hashCode) + (search.hashCode) + (sidecar.hashCode) + @@ -113,7 +119,7 @@ class ServerFeaturesDto { (trash.hashCode); @override - String toString() => 'ServerFeaturesDto[configFile=$configFile, duplicateDetection=$duplicateDetection, email=$email, facialRecognition=$facialRecognition, importFaces=$importFaces, map=$map, oauth=$oauth, oauthAutoLaunch=$oauthAutoLaunch, ocr=$ocr, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, trash=$trash]'; + String toString() => 'ServerFeaturesDto[configFile=$configFile, duplicateDetection=$duplicateDetection, email=$email, facialRecognition=$facialRecognition, importFaces=$importFaces, map=$map, oauth=$oauth, oauthAutoLaunch=$oauthAutoLaunch, ocr=$ocr, passwordLogin=$passwordLogin, realtimeTranscoding=$realtimeTranscoding, reverseGeocoding=$reverseGeocoding, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, trash=$trash]'; Map toJson() { final json = {}; @@ -127,6 +133,7 @@ class ServerFeaturesDto { json[r'oauthAutoLaunch'] = this.oauthAutoLaunch; json[r'ocr'] = this.ocr; json[r'passwordLogin'] = this.passwordLogin; + json[r'realtimeTranscoding'] = this.realtimeTranscoding; json[r'reverseGeocoding'] = this.reverseGeocoding; json[r'search'] = this.search; json[r'sidecar'] = this.sidecar; @@ -154,6 +161,7 @@ class ServerFeaturesDto { oauthAutoLaunch: mapValueOfType(json, r'oauthAutoLaunch')!, ocr: mapValueOfType(json, r'ocr')!, passwordLogin: mapValueOfType(json, r'passwordLogin')!, + realtimeTranscoding: mapValueOfType(json, r'realtimeTranscoding')!, reverseGeocoding: mapValueOfType(json, r'reverseGeocoding')!, search: mapValueOfType(json, r'search')!, sidecar: mapValueOfType(json, r'sidecar')!, @@ -216,6 +224,7 @@ class ServerFeaturesDto { 'oauthAutoLaunch', 'ocr', 'passwordLogin', + 'realtimeTranscoding', 'reverseGeocoding', 'search', 'sidecar', diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 68d04a665a..d9087c375d 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -21682,6 +21682,10 @@ "description": "Whether password login is enabled", "type": "boolean" }, + "realtimeTranscoding": { + "description": "Whether real-time transcoding is enabled", + "type": "boolean" + }, "reverseGeocoding": { "description": "Whether reverse geocoding is enabled", "type": "boolean" @@ -21714,6 +21718,7 @@ "oauthAutoLaunch", "ocr", "passwordLogin", + "realtimeTranscoding", "reverseGeocoding", "search", "sidecar", diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.ts index 85b859ac1d..163558e6a6 100644 --- a/packages/sdk/src/fetch-client.ts +++ b/packages/sdk/src/fetch-client.ts @@ -1993,6 +1993,8 @@ export type ServerFeaturesDto = { ocr: boolean; /** Whether password login is enabled */ passwordLogin: boolean; + /** Whether real-time transcoding is enabled */ + realtimeTranscoding: boolean; /** Whether reverse geocoding is enabled */ reverseGeocoding: boolean; /** Whether search is enabled */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 329203c36b..debdb79b93 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -820,6 +820,12 @@ importers: happy-dom: specifier: ^20.0.0 version: 20.9.0 + hls-video-element: + specifier: ^1.5.11 + version: 1.5.11 + hls.js: + specifier: ^1.6.16 + version: 1.6.16 intl-messageformat: specifier: ^11.0.0 version: 11.2.6 @@ -6947,6 +6953,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + custom-media-element@1.4.6: + resolution: {integrity: sha512-/HRYqJOa1ob5ik4q7FIJVYxTJCFs/FL3+cQPAJjUf2uiqrDEzbTgB315gQ2rG8oK3w094W9m5tcB8S5Qah+caA==} + cytoscape-cose-bilkent@4.1.0: resolution: {integrity: sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==} peerDependencies: @@ -8271,6 +8280,12 @@ packages: history@4.10.1: resolution: {integrity: sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==} + hls-video-element@1.5.11: + resolution: {integrity: sha512-tJJ65/52CDxj8XFyIve6zT9nVVdUIc6mqvKR25X0ycPKHk07rpjp4xxVteeCefDUBSf/tFLhlICFmn3KWj37xA==} + + hls.js@1.6.16: + resolution: {integrity: sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==} + hogan.js@3.0.2: resolution: {integrity: sha512-RqGs4wavGYJWE07t35JQccByczmNUXQT0E12ZYV1VKYu5UiAU9lsos/yBAcf840+zrUQQxgVduCR5/B8nNtibg==} hasBin: true @@ -9275,6 +9290,9 @@ packages: media-chrome@4.19.0: resolution: {integrity: sha512-HWhDTwts+BSbdPkkB1VsJXp5kvL0IxY7xFT5tBwliM2+89kTPVTnHnev+9it2f9PweANjT/C8/C/S0PW9oyZbA==} + media-tracks@0.3.5: + resolution: {integrity: sha512-l54rkKXlLBt3ob3zOLWHcnjvwUmX5bNEZ70igyapOZZC9imzqBmq1oz8p2roiV04KhjblFIi2hetLPF1oYVLRA==} + media-typer@0.3.0: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} @@ -19869,6 +19887,8 @@ snapshots: csstype@3.2.3: {} + custom-media-element@1.4.6: {} + cytoscape-cose-bilkent@4.1.0(cytoscape@3.33.4): dependencies: cose-base: 1.0.3 @@ -21558,6 +21578,14 @@ snapshots: tiny-warning: 1.0.3 value-equal: 1.0.1 + hls-video-element@1.5.11: + dependencies: + custom-media-element: 1.4.6 + hls.js: 1.6.16 + media-tracks: 0.3.5 + + hls.js@1.6.16: {} + hogan.js@3.0.2: dependencies: mkdirp: 0.3.0 @@ -22661,6 +22689,8 @@ snapshots: transitivePeerDependencies: - react + media-tracks@0.3.5: {} + media-typer@0.3.0: {} media-typer@1.1.0: {} diff --git a/server/src/dtos/server.dto.ts b/server/src/dtos/server.dto.ts index c770ec12ca..6370557785 100644 --- a/server/src/dtos/server.dto.ts +++ b/server/src/dtos/server.dto.ts @@ -144,6 +144,7 @@ const ServerFeaturesSchema = z search: z.boolean().describe('Whether search is enabled'), email: z.boolean().describe('Whether email notifications are enabled'), ocr: z.boolean().describe('Whether OCR is enabled'), + realtimeTranscoding: z.boolean().describe('Whether real-time transcoding is enabled'), }) .meta({ id: 'ServerFeaturesDto' }); diff --git a/server/src/services/server.service.spec.ts b/server/src/services/server.service.spec.ts index 6e1187a900..e02945d015 100644 --- a/server/src/services/server.service.spec.ts +++ b/server/src/services/server.service.spec.ts @@ -148,6 +148,7 @@ describe(ServerService.name, () => { configFile: false, trash: true, email: false, + realtimeTranscoding: false, }); expect(mocks.systemMetadata.get).toHaveBeenCalled(); }); diff --git a/server/src/services/server.service.ts b/server/src/services/server.service.ts index ff9e90f7e0..aeeb41fcb0 100644 --- a/server/src/services/server.service.ts +++ b/server/src/services/server.service.ts @@ -86,7 +86,7 @@ export class ServerService extends BaseService { } async getFeatures(): Promise { - const { reverseGeocoding, metadata, map, machineLearning, trash, oauth, passwordLogin, notifications } = + const { reverseGeocoding, metadata, map, machineLearning, trash, oauth, passwordLogin, notifications, ffmpeg } = await this.getConfig({ withCache: false }); const { configFile } = this.configRepository.getEnv(); @@ -106,6 +106,7 @@ export class ServerService extends BaseService { passwordLogin: passwordLogin.enabled, configFile: !!configFile, email: notifications.smtp.enabled, + realtimeTranscoding: ffmpeg.realtime.enabled, }; } diff --git a/web/package.json b/web/package.json index 2fb37daaed..36f5cf9647 100644 --- a/web/package.json +++ b/web/package.json @@ -46,6 +46,8 @@ "geojson": "^0.5.0", "handlebars": "^4.7.8", "happy-dom": "^20.0.0", + "hls-video-element": "^1.5.11", + "hls.js": "^1.6.16", "intl-messageformat": "^11.0.0", "justified-layout": "^4.1.0", "lodash-es": "^4.17.21", diff --git a/web/src/lib/components/asset-viewer/VideoNativeViewer.svelte b/web/src/lib/components/asset-viewer/VideoNativeViewer.svelte index 295c5842a0..8f84466295 100644 --- a/web/src/lib/components/asset-viewer/VideoNativeViewer.svelte +++ b/web/src/lib/components/asset-viewer/VideoNativeViewer.svelte @@ -5,7 +5,7 @@ import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte'; import { castManager } from '$lib/managers/cast-manager.svelte'; import { autoPlayVideo, lang, loopVideo as loopVideoPreference } from '$lib/stores/preferences.store'; - import { getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils'; + import { getAssetHlsSessionUrl, getAssetHlsUrl, getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils'; import { AssetMediaSize, type AssetResponseDto } from '@immich/sdk'; import { Icon, LoadingSpinner } from '@immich/ui'; import { @@ -21,6 +21,9 @@ mdiVolumeMedium, mdiVolumeMute, } from '@mdi/js'; + import Hls, { AbrController, Events, type FragLoadedData, type FragLoadingData, type HlsConfig } from 'hls.js'; + import 'hls-video-element'; + import type HlsVideoElement from 'hls-video-element'; import 'media-chrome/media-control-bar'; import 'media-chrome/media-controller'; import 'media-chrome/media-fullscreen-button'; @@ -28,9 +31,10 @@ import 'media-chrome/media-play-button'; import 'media-chrome/media-playback-rate-button'; import 'media-chrome/media-time-display'; - import 'media-chrome/media-time-range'; + import './immich-time-range'; import 'media-chrome/media-volume-range'; import 'media-chrome/menu/media-playback-rate-menu'; + import 'media-chrome/menu/media-rendition-menu'; import 'media-chrome/menu/media-settings-menu'; import 'media-chrome/menu/media-settings-menu-button'; import 'media-chrome/menu/media-settings-menu-item'; @@ -38,6 +42,8 @@ import { useSwipe, type SwipeCustomEvent } from 'svelte-gestures'; import { t } from 'svelte-i18n'; import { fade } from 'svelte/transition'; + import { featureFlagsManager } from '$lib/managers/feature-flags-manager.svelte'; + import { mediaCapabilitiesManager } from '$lib/managers/media-capabilities-manager.svelte'; interface Props { asset: AssetResponseDto; @@ -69,14 +75,155 @@ let videoPlayer: HTMLVideoElement | undefined = $state(); let isLoading = $state(true); - let assetFileUrl = $derived( - playOriginalVideo - ? getAssetMediaUrl({ id: assetId, size: AssetMediaSize.Original, cacheKey }) - : getAssetPlaybackUrl({ id: assetId, cacheKey }), - ); + let assetFileUrl = $derived.by(() => { + if (featureFlagsManager.value.realtimeTranscoding) { + return getAssetHlsUrl(assetId); + } + + if (playOriginalVideo) { + return getAssetMediaUrl({ id: assetId, size: AssetMediaSize.Original, cacheKey }); + } + + return getAssetPlaybackUrl({ id: assetId, cacheKey }); + }); const aspectRatio = $derived(asset.width && asset.height ? `${asset.width} / ${asset.height}` : undefined); let showVideo = $state(false); let hasFocused = $state(false); + let activeSession: { assetId: string; id: string } | undefined; + let rebuildCount = 0; + + const MAX_REBUILDS = 1; + const SESSION_ID_REGEX = /\/video\/stream\/([0-9a-f-]{36})\//; + + // hls.js can abandon fetching an in-flight fragment if it thinks it'll take too long, in which case + // it emergency switches to a different variant. This extends the delay even further due to + // cold starting another transcode, so let the fragment finish and have steady ABR decide the next level. + // + // It can also emergency switch between fragments: while a switch's first segment is still loading, + // it can run out of buffer and drop to a lower level for just one segment before continuing at the switched quality. + // This can cause multiple redundant transcoding restarts when it occurs. + // Hold the committed level until its first fragment lands, then resume normal ABR. + class NoAbandonAbrController extends AbrController { + private switchTarget = -1; + + protected override onFragLoading(_event: Events.FRAG_LOADING, data: FragLoadingData) { + if (data.frag.sn === 'initSegment') { + this.switchTarget = data.frag.level; + } + } + + protected override onFragLoaded(event: Events.FRAG_LOADED, data: FragLoadedData) { + if (data.frag.sn !== 'initSegment') { + this.switchTarget = -1; + } + super.onFragLoaded(event, data); + } + + override get nextAutoLevel(): number { + const level = super.nextAutoLevel; + const target = this.hls.levels[this.switchTarget]; + // Hold the committed level, but only while hls.js still considers it healthy. + if (target && level < this.switchTarget && target.loadError === 0 && target.fragmentError === 0) { + return this.switchTarget; + } + return level; + } + + override set nextAutoLevel(level: number) { + super.nextAutoLevel = level; + } + } + + const hlsConfig: Partial = { + abrController: NoAbandonAbrController, + highBufferWatchdogPeriod: 10, + detectStallWithCurrentTimeMs: 10_000, + maxBufferHole: 0.5, + maxBufferLength: 30, + maxMaxBufferLength: 60, + fragLoadPolicy: { + default: { + maxTimeToFirstByteMs: 30_000, + maxLoadTimeMs: 60_000, + timeoutRetry: { maxNumRetry: 5, retryDelayMs: 100, maxRetryDelayMs: 0 }, + errorRetry: { maxNumRetry: 3, retryDelayMs: 1000, maxRetryDelayMs: 8000 }, + }, + }, + useMediaCapabilities: false, + }; + + const releaseSession = () => { + const session = activeSession; + if (!session) { + return; + } + activeSession = undefined; + const url = getAssetHlsSessionUrl(session.assetId, session.id); + void fetch(url, { method: 'DELETE' }).catch(() => console.warn('Failed to release HLS session', session)); + }; + + const isHlsElement = (el: HTMLVideoElement | undefined): el is HlsVideoElement => { + return el?.tagName === 'HLS-VIDEO'; + }; + + const wireHlsListeners = (el: HlsVideoElement, assetId: string, resumeTime?: number) => { + const api = el.api; + if (!api) { + return; + } + + // This is a hack to make the rendition menu use `api.currentLevel` instead of `api.nextLevel`. + // `api.nextLevel` makes the player request the next segment followed by the current segment. + // That backward request causes the server to restart transcoding for no reason. + Object.defineProperty(api, 'nextLevel', { + configurable: true, + get: () => api.currentLevel, + set: (level: number) => { + api.currentLevel = level; + }, + }); + + // eslint-disable-next-line @typescript-eslint/no-misused-promises + api.on(Hls.Events.MANIFEST_PARSED, async () => { + // Defer hls.js's first fragment load until we filter out suboptimal variants + api.stopLoad(); + const id = api.levels[0]?.url[0]?.match(SESSION_ID_REGEX)?.[1]; + if (id) { + activeSession = { assetId, id }; + } + + const keep = await mediaCapabilitiesManager.efficientLevels(api.levels); + for (let i = api.levels.length - 1; i >= 0; i--) { + if (!keep.has(i)) { + api.removeLevel(i); + } + } + + api.startLoad(resumeTime); + }); + + api.on(Hls.Events.FRAG_LOADED, () => (rebuildCount = 0)); + + api.on(Hls.Events.ERROR, (_, data) => { + // 404 on a fragment can mean the server-side session has expired. Refetch + // master for a new session, but give up if it still 404s. + if ( + !data.fatal || + data.details !== Hls.ErrorDetails.FRAG_LOAD_ERROR || + data.response?.code !== 404 || + rebuildCount++ >= MAX_REBUILDS + ) { + console.error('HLS error', JSON.stringify(data)); + return; + } + console.warn('Error loading segment, starting new session'); + activeSession = undefined; + resumeTime = el.currentTime; + el.load(); + // wireHlsListeners must run after el.api is repopulated. + queueMicrotask(() => wireHlsListeners(el, assetId, resumeTime)); + }); + }; onMount(() => { showVideo = true; @@ -84,10 +231,31 @@ $effect(() => { // reactive on `assetFileUrl` changes - if (assetFileUrl) { + if (videoPlayer && assetFileUrl) { hasFocused = false; - videoPlayer?.load(); + rebuildCount = 0; + releaseSession(); + if (isHlsElement(videoPlayer)) { + videoPlayer.config = hlsConfig; + videoPlayer.src = assetFileUrl; + const el = videoPlayer; + queueMicrotask(() => wireHlsListeners(el, assetId)); + } else { + videoPlayer.load(); + } } + return releaseSession; + }); + + const onPagehide = (event: PageTransitionEvent) => { + if (!event.persisted) { + releaseSession(); + } + }; + + $effect(() => { + window.addEventListener('pagehide', onPagehide); + return () => window.removeEventListener('pagehide', onPagehide); }); onDestroy(() => { @@ -144,6 +312,10 @@ videoPlayer?.pause(); } }); + + // The time is only refreshed on HLS fragment decode by default, + // so manually emit events on seek to update it immediately. + const onSeeking = (event: Event) => event.currentTarget?.dispatchEvent(new Event('timeupdate')); {#if showVideo} @@ -172,27 +344,51 @@ style:aspect-ratio={aspectRatio} defaultduration={asset.duration! / 1000} > - + {#if featureFlagsManager.value.realtimeTranscoding} + handleCanPlay(e.currentTarget as HTMLVideoElement)} + onended={onVideoEnded} + onseeking={onSeeking} + onplaying={(e: Event) => { + if (!hasFocused) { + (e.currentTarget as HTMLElement).focus(); + hasFocused = true; + } + }} + onclose={onClose} + poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview, cacheKey })} + > + {:else} + + {/if} {#if extendedControls} {/if} @@ -238,7 +444,7 @@ {/if} - + @@ -248,7 +454,7 @@ {/if} - {#if assetViewerManager.isFaceEditMode} + {#if assetViewerManager.isFaceEditMode && videoPlayer} {/if} {/if} @@ -291,12 +497,12 @@ font-variant-numeric: tabular-nums; } - media-time-range, + immich-time-range, media-volume-range { --media-control-hover-background: none; } - media-time-range:hover, + immich-time-range:hover, media-volume-range:hover { --media-range-thumb-opacity: 1; } diff --git a/web/src/lib/components/asset-viewer/immich-time-range.ts b/web/src/lib/components/asset-viewer/immich-time-range.ts new file mode 100644 index 0000000000..a3de131e73 --- /dev/null +++ b/web/src/lib/components/asset-viewer/immich-time-range.ts @@ -0,0 +1,54 @@ +import { MediaUIEvents } from 'media-chrome/constants'; +import MediaTimeRange from 'media-chrome/media-time-range'; + +const COMMIT_DELAY_MS = 750; + +/** Custom MediaTimeRange that only seeks after pointer release to avoid hammering the server. + * Keyboard input uses timed debouncing instead since there's no release event. */ +class ImmichTimeRange extends MediaTimeRange { + private seeking = false; + private pending: number | undefined; + private idleTimer: ReturnType | undefined; + + override connectedCallback() { + super.connectedCallback(); + this.addEventListener('pointerdown', this.hold); + this.addEventListener('keydown', this.hold); + this.addEventListener('pointerup', this.release); + this.addEventListener('pointercancel', this.release); + this.addEventListener(MediaUIEvents.MEDIA_SEEK_REQUEST, this.intercept, { capture: true }); + } + + private hold(event: Event) { + if (event instanceof KeyboardEvent) { + if (!this.keysUsed.includes(event.key)) { + return; + } + clearTimeout(this.idleTimer); + this.idleTimer = setTimeout(this.release, COMMIT_DELAY_MS); + } + this.seeking = true; + } + + private intercept(event: Event) { + if (!this.seeking) { + return; // not mid-scrub, or this is the request we replay in release() + } + this.pending = (event as CustomEvent).detail; + event.stopImmediatePropagation(); + } + + private release() { + clearTimeout(this.idleTimer); + this.seeking = false; + if (this.pending !== undefined) { + const detail = this.pending; + this.pending = undefined; + this.dispatchEvent(new CustomEvent(MediaUIEvents.MEDIA_SEEK_REQUEST, { bubbles: true, composed: true, detail })); + } + } +} + +if (!globalThis.customElements.get('immich-time-range')) { + globalThis.customElements.define('immich-time-range', ImmichTimeRange); +} diff --git a/web/src/lib/managers/media-capabilities-manager.svelte.ts b/web/src/lib/managers/media-capabilities-manager.svelte.ts new file mode 100644 index 0000000000..ccabc6680d --- /dev/null +++ b/web/src/lib/managers/media-capabilities-manager.svelte.ts @@ -0,0 +1,92 @@ +export type Level = { videoCodec?: string; width: number; height: number; bitrate: number; frameRate: number }; + +export const DEFAULT_DECODING_INFO: MediaCapabilitiesDecodingInfo = { + powerEfficient: true, + smooth: true, + supported: true, + keySystemAccess: null, +}; + +class MediaCapabilitiesManager { + private cache = new Map>(); + + init() { + for (const level of [ + { videoCodec: 'av01.0.04M.08', width: 854, height: 480, bitrate: 1_000_000, frameRate: 60 }, + { videoCodec: 'hvc1.1.6.L90.B0', width: 854, height: 480, bitrate: 1_200_000, frameRate: 60 }, + { videoCodec: 'av01.0.08M.08', width: 1280, height: 720, bitrate: 2_000_000, frameRate: 60 }, + { videoCodec: 'hvc1.1.6.L93.B0', width: 1280, height: 720, bitrate: 2_500_000, frameRate: 60 }, + { videoCodec: 'av01.0.09M.08', width: 1920, height: 1080, bitrate: 4_000_000, frameRate: 60 }, + { videoCodec: 'hvc1.1.6.L120.B0', width: 1920, height: 1080, bitrate: 4_500_000, frameRate: 60 }, + { videoCodec: 'av01.0.12M.08', width: 2560, height: 1440, bitrate: 7_000_000, frameRate: 60 }, + { videoCodec: 'hvc1.2.4.L150.B0', width: 2560, height: 1440, bitrate: 8_000_000, frameRate: 60 }, + ]) { + this.cache.set(this.cacheKey(level), this.queryDecodingInfo(level)); + } + + for (const level of [ + { videoCodec: 'avc1.64001e', width: 854, height: 480, bitrate: 2_500_000, frameRate: 60 }, + { videoCodec: 'avc1.64001f', width: 1280, height: 720, bitrate: 5_000_000, frameRate: 60 }, + { videoCodec: 'avc1.640028', width: 1920, height: 1080, bitrate: 8_000_000, frameRate: 60 }, + { videoCodec: 'avc1.640032', width: 2560, height: 1440, bitrate: 16_000_000, frameRate: 60 }, + ]) { + this.cache.set(this.cacheKey(level), Promise.resolve(DEFAULT_DECODING_INFO)); + } + } + + async efficientLevels(levels: Level[]) { + const decodingInfo = await Promise.all(levels.map((level) => this.decodingInfo(level))); + // eslint-disable-next-line svelte/prefer-svelte-reactivity + const lowestBitrateByHeight = new Map(); + for (let i = 0; i < levels.length; i++) { + if (!decodingInfo[i].powerEfficient) { + continue; + } + + const { bitrate, height } = levels[i]; + const cur = lowestBitrateByHeight.get(height); + if (cur === undefined || bitrate < levels[cur].bitrate) { + lowestBitrateByHeight.set(height, i); + } + } + + return new Set(lowestBitrateByHeight.values()); + } + + decodingInfo(level: Level) { + const key = this.cacheKey(level); + const existing = this.cache.get(key); + if (existing) { + return existing; + } + const promise = this.queryDecodingInfo(level); + this.cache.set(key, promise); + return promise; + } + + private async queryDecodingInfo(level: Level) { + try { + return await navigator.mediaCapabilities.decodingInfo({ + type: 'media-source', + video: { + contentType: `video/mp4; codecs="${level.videoCodec}"`, + width: level.width, + height: level.height, + bitrate: level.bitrate, + framerate: level.frameRate, + }, + }); + } catch { + return DEFAULT_DECODING_INFO; + } + } + + private cacheKey({ videoCodec, width, height, frameRate }: Level) { + const resolution = Math.min(width, height); + const fpsBucket = Math.trunc(frameRate / 61) * 60; + return `${videoCodec}|${resolution}|${fpsBucket}`; + } +} + +export const mediaCapabilitiesManager = new MediaCapabilitiesManager(); +mediaCapabilitiesManager.init(); diff --git a/web/src/lib/utils.ts b/web/src/lib/utils.ts index 9af417bb19..4ff3564ffa 100644 --- a/web/src/lib/utils.ts +++ b/web/src/lib/utils.ts @@ -244,6 +244,14 @@ export const getAssetPlaybackUrl = (options: AssetUrlOptions) => { return createUrl(getAssetPlaybackPath(id), { ...authManager.params, c }); }; +export const getAssetHlsUrl = (id: string) => { + return createUrl(`/assets/${id}/video/stream/main.m3u8`, authManager.params); +}; + +export const getAssetHlsSessionUrl = (id: string, sessionId: string) => { + return createUrl(`/assets/${id}/video/stream/${sessionId}`, authManager.params); +}; + export const getProfileImageUrl = (user: UserResponseDto) => createUrl(getUserProfileImagePath(user.id), { updatedAt: user.profileChangedAt });