diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index a8e96709ff..12bbeabe34 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -39,6 +39,7 @@ type ProgressEvent = { export type ExtractResult = { buffer: Buffer; format: RawExtractedFormat; + dimensions: ImageDimensions; }; @Injectable() @@ -55,28 +56,28 @@ export class MediaRepository { async extract(input: string): Promise { try { const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw2', input); - return { buffer, format: RawExtractedFormat.Jpeg }; + return { buffer, format: RawExtractedFormat.Jpeg, dimensions: await this.getImageDimensions(buffer) }; } catch (error: any) { this.logger.debug(`Could not extract JpgFromRaw2 buffer from image, trying JPEG from RAW next: ${error}`); } try { const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw', input); - return { buffer, format: RawExtractedFormat.Jpeg }; + return { buffer, format: RawExtractedFormat.Jpeg, dimensions: await this.getImageDimensions(buffer) }; } catch (error: any) { this.logger.debug(`Could not extract JPEG buffer from image, trying PreviewJXL next: ${error}`); } try { const buffer = await exiftool.extractBinaryTagToBuffer('PreviewJXL', input); - return { buffer, format: RawExtractedFormat.Jxl }; + return { buffer, format: RawExtractedFormat.Jxl, dimensions: await this.getImageDimensions(buffer) }; } catch (error: any) { this.logger.debug(`Could not extract PreviewJXL buffer from image, trying PreviewImage next: ${error}`); } try { const buffer = await exiftool.extractBinaryTagToBuffer('PreviewImage', input); - return { buffer, format: RawExtractedFormat.Jpeg }; + return { buffer, format: RawExtractedFormat.Jpeg, dimensions: await this.getImageDimensions(buffer) }; } catch (error: any) { this.logger.debug(`Could not extract preview buffer from image: ${error}`); return null; @@ -121,19 +122,15 @@ export class MediaRepository { } } - async copyTagGroup(tagGroup: string, source: string, target: string): Promise { + async writeTags(tags: WriteTags, output: string): Promise { try { - await exiftool.write( - target, - {}, - { - ignoreMinorErrors: true, - writeArgs: ['-TagsFromFile', source, `-${tagGroup}:all>${tagGroup}:all`, '-overwrite_original'], - }, - ); + await exiftool.write(output, tags, { + ignoreMinorErrors: true, + writeArgs: ['-overwrite_original'], + }); return true; } catch (error: any) { - this.logger.warn(`Could not copy tag data to image: ${error.message}`); + this.logger.warn(`Could not write tags to image: ${error.message}`); return false; } } @@ -273,7 +270,7 @@ export class MediaRepository { }); } - async getImageDimensions(input: string | Buffer): Promise { + private async getImageDimensions(input: string | Buffer): Promise { const { width = 0, height = 0 } = await sharp(input).metadata(); return { width, height }; } diff --git a/server/src/repositories/metadata.repository.ts b/server/src/repositories/metadata.repository.ts index 1334d1220f..1a7b4afc59 100644 --- a/server/src/repositories/metadata.repository.ts +++ b/server/src/repositories/metadata.repository.ts @@ -72,6 +72,21 @@ export interface ImmichTags extends Omit { AndroidMake?: string; AndroidModel?: string; + + UsePanoramaViewer?: boolean; + ProjectionType?: string; + PoseHeadingDegrees?: number; + PosePitchDegrees?: number; + PoseRollDegrees?: number; + InitialViewHeadingDegrees?: number; + InitialViewPitchDegrees?: number; + InitialViewRollDegrees?: number; + CroppedAreaImageWidthPixels?: number; + CroppedAreaImageHeightPixels?: number; + FullPanoWidthPixels?: number; + FullPanoHeightPixels?: number; + CroppedAreaLeftPixels?: number; + CroppedAreaTopPixels?: number; } @Injectable() diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 8617930534..fb3347a770 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -599,8 +599,11 @@ describe(MediaService.name, () => { }); it('should extract embedded image if enabled and available', async () => { - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jpeg, + dimensions: { width: 3840, height: 2160 }, + }); mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng); @@ -615,8 +618,11 @@ describe(MediaService.name, () => { }); it('should resize original image if embedded image is too small', async () => { - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 1000, height: 1000 }); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jpeg, + dimensions: { width: 1000, height: 1000 }, + }); mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng); @@ -641,7 +647,6 @@ describe(MediaService.name, () => { processInvalidImages: false, size: 1440, }); - expect(mocks.media.getImageDimensions).not.toHaveBeenCalled(); }); it('should resize original image if embedded image extraction is not enabled', async () => { @@ -657,7 +662,6 @@ describe(MediaService.name, () => { processInvalidImages: false, size: 1440, }); - expect(mocks.media.getImageDimensions).not.toHaveBeenCalled(); }); it('should process invalid images if enabled', async () => { @@ -691,7 +695,6 @@ describe(MediaService.name, () => { expect.objectContaining({ processInvalidImages: false }), ); - expect(mocks.media.getImageDimensions).not.toHaveBeenCalled(); vi.unstubAllEnvs(); }); @@ -699,8 +702,11 @@ describe(MediaService.name, () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true, format: ImageFormat.Webp }, extractEmbedded: true }, }); - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jpeg, + dimensions: { width: 3840, height: 2160 }, + }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng); await sut.handleGenerateThumbnails({ id: assetStub.image.id }); @@ -731,8 +737,11 @@ describe(MediaService.name, () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true, format: ImageFormat.Webp }, extractEmbedded: true }, }); - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jxl }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jxl, + dimensions: { width: 3840, height: 2160 }, + }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng); await sut.handleGenerateThumbnails({ id: assetStub.image.id }); @@ -771,8 +780,11 @@ describe(MediaService.name, () => { it('should generate full-size preview directly from RAW images when extractEmbedded is false', async () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true }, extractEmbedded: false } }); - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jpeg, + dimensions: { width: 3840, height: 2160 }, + }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng); await sut.handleGenerateThumbnails({ id: assetStub.image.id }); @@ -811,8 +823,11 @@ describe(MediaService.name, () => { it('should generate full-size preview from non-web-friendly images', async () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } }); - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jpeg, + dimensions: { width: 3840, height: 2160 }, + }); // HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari. mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageHif); @@ -840,8 +855,11 @@ describe(MediaService.name, () => { it('should skip generating full-size preview for web-friendly images', async () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } }); - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jpeg, + dimensions: { width: 3840, height: 2160 }, + }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.image); await sut.handleGenerateThumbnails({ id: assetStub.image.id }); @@ -863,9 +881,17 @@ describe(MediaService.name, () => { it('should always generate full-size preview from non-web-friendly panoramas', async () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } }); - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); - mocks.media.copyTagGroup.mockResolvedValue(true); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jpeg, + dimensions: { width: 3840, height: 2160 }, + }); + mocks.metadata.readTags.mockResolvedValue({ + ProjectionType: 'equirectangular', + PoseHeadingDegrees: 127, + FullPanoWidthPixels: 3840, + FullPanoHeightPixels: 2160, + }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.panoramaTif); @@ -892,10 +918,14 @@ describe(MediaService.name, () => { expect.any(String), ); - expect(mocks.media.copyTagGroup).toHaveBeenCalledTimes(2); - expect(mocks.media.copyTagGroup).toHaveBeenCalledWith( - 'XMP-GPano', - assetStub.panoramaTif.originalPath, + expect(mocks.media.writeTags).toHaveBeenCalledTimes(2); + expect(mocks.media.writeTags).toHaveBeenCalledWith( + { + ProjectionType: 'equirectangular', + PoseHeadingDegrees: 127, + FullPanoWidthPixels: 2560, + FullPanoHeightPixels: 1440, + }, expect.any(String), ); }); @@ -904,8 +934,11 @@ describe(MediaService.name, () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true, format: ImageFormat.Webp, quality: 90 } }, }); - mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.extract.mockResolvedValue({ + buffer: extractedBuffer, + format: RawExtractedFormat.Jpeg, + dimensions: { width: 3840, height: 2160 }, + }); // HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari. mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageHif); @@ -1190,9 +1223,8 @@ describe(MediaService.name, () => { const extracted = Buffer.from(''); const data = Buffer.from(''); const info = { width: 2160, height: 3840 } as OutputInfo; - mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg }); + mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg, dimensions: info }); mocks.media.decodeImage.mockResolvedValue({ data, info }); - mocks.media.getImageDimensions.mockResolvedValue(info); await expect(sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id })).resolves.toBe( JobStatus.Success, @@ -1268,8 +1300,7 @@ describe(MediaService.name, () => { const data = Buffer.from(''); const info = { width: 1000, height: 1000 } as OutputInfo; mocks.media.decodeImage.mockResolvedValue({ data, info }); - mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue(info); + mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg, dimensions: info }); await expect(sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id })).resolves.toBe( JobStatus.Success, diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 9027e89d66..34eb39f9e1 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -47,6 +47,26 @@ interface UpsertFileOptions { path: string; } +const PANORAMA_CONSTANTS = [ + 'UsePanoramaViewer', + 'ProjectionType', + 'PoseHeadingDegrees', + 'PosePitchDegrees', + 'PoseRollDegrees', + 'InitialViewHeadingDegrees', + 'InitialViewPitchDegrees', + 'InitialViewRollDegrees', +] as const; + +const PANORAMA_SCALABLES = [ + 'CroppedAreaImageWidthPixels', + 'CroppedAreaImageHeightPixels', + 'FullPanoWidthPixels', + 'FullPanoHeightPixels', + 'CroppedAreaLeftPixels', + 'CroppedAreaTopPixels', +] as const; + @Injectable() export class MediaService extends BaseService { videoInterfaces: VideoInterfaces = { dri: [], mali: false }; @@ -237,7 +257,7 @@ export class MediaService extends BaseService { private async extractImage(originalPath: string, minSize: number) { let extracted = await this.mediaRepository.extract(originalPath); - if (extracted && !(await this.shouldUseExtractedImage(extracted.buffer, minSize))) { + if (extracted && !this.shouldUseExtractedImage(extracted.dimensions, minSize)) { extracted = null; } @@ -295,14 +315,21 @@ export class MediaService extends BaseService { ]; let fullsizePath: string | undefined; + let fullsizeSize: number | undefined; + const originalSize = + asset.exifInfo.exifImageWidth && asset.exifInfo.exifImageHeight + ? Math.min(asset.exifInfo.exifImageWidth, asset.exifInfo.exifImageHeight) + : undefined; if (convertFullsize) { // convert a new fullsize image from the same source as the thumbnail fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FullSize, image.fullsize.format); + fullsizeSize = originalSize; const fullsizeOptions = { format: image.fullsize.format, quality: image.fullsize.quality, ...thumbnailOptions }; promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizePath)); } else if (generateFullsize && extracted && extracted.format === RawExtractedFormat.Jpeg) { fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FullSize, extracted.format); + fullsizeSize = Math.min(extracted.dimensions.width, extracted.dimensions.height); this.storageCore.ensureFolders(fullsizePath); // Write the buffer to disk with essential EXIF data @@ -318,19 +345,55 @@ export class MediaService extends BaseService { const outputs = await Promise.all(promises); - if (asset.exifInfo.projectionType === 'EQUIRECTANGULAR') { - const promises = [ - this.mediaRepository.copyTagGroup('XMP-GPano', asset.originalPath, previewPath), - fullsizePath - ? this.mediaRepository.copyTagGroup('XMP-GPano', asset.originalPath, fullsizePath) - : Promise.resolve(), - ]; - await Promise.all(promises); + if (asset.exifInfo.projectionType === 'EQUIRECTANGULAR' && originalSize) { + await this.copyPanoramaMetadataToThumbnails( + asset.originalPath, + originalSize, + previewPath, + image.preview.size, + fullsizePath, + fullsizeSize, + ); } return { previewPath, thumbnailPath, fullsizePath, thumbhash: outputs[0] as Buffer }; } + private async copyPanoramaMetadataToThumbnails( + originalPath: string, + originalSize: number, + previewPath: string, + previewSize: number, + fullsizePath?: string, + fullsizeSize?: number, + ) { + const originalTags = await this.metadataRepository.readTags(originalPath); + + const scaleAndWriteData = async (thumbnailPath: string, scaleRatio: number) => { + const newTags = {} as Record; + + for (const key of PANORAMA_CONSTANTS) { + if (key in originalTags && originalTags[key]) { + newTags[key] = originalTags[key]; + } + } + for (const key of PANORAMA_SCALABLES) { + if (key in originalTags && originalTags[key]) { + newTags[key] = Math.round(originalTags[key] * scaleRatio); + } + } + + return this.mediaRepository.writeTags(newTags, thumbnailPath); + }; + + const promises = [ + // preview size is min(preview size, original size) so do the same for pano pixel adjustment + scaleAndWriteData(previewPath, Math.min(1, previewSize / originalSize)), + fullsizePath && fullsizeSize ? scaleAndWriteData(fullsizePath, fullsizeSize / originalSize) : Promise.resolve(), + ]; + await Promise.all(promises); + } + @OnJob({ name: JobName.PersonGenerateThumbnail, queue: QueueName.ThumbnailGeneration }) async handleGeneratePersonThumbnail({ id }: JobOf): Promise { const { machineLearning, metadata, image } = await this.getConfig({ withCache: true }); @@ -680,8 +743,8 @@ export class MediaService extends BaseService { } } - private async shouldUseExtractedImage(extractedPathOrBuffer: string | Buffer, targetSize: number) { - const { width, height } = await this.mediaRepository.getImageDimensions(extractedPathOrBuffer); + private shouldUseExtractedImage(extractedDimensions: ImageDimensions, targetSize: number) { + const { width, height } = extractedDimensions; const extractedSize = Math.min(width, height); return extractedSize >= targetSize; } diff --git a/server/test/fixtures/asset.stub.ts b/server/test/fixtures/asset.stub.ts index f5935d5d0e..1be1e92e6f 100644 --- a/server/test/fixtures/asset.stub.ts +++ b/server/test/fixtures/asset.stub.ts @@ -892,6 +892,8 @@ export const assetStub = { exifInfo: { fileSizeInByte: 5000, projectionType: 'EQUIRECTANGULAR', + exifImageHeight: 2160, + exifImageWidth: 3840, } as Exif, duplicateId: null, isOffline: false, diff --git a/server/test/repositories/media.repository.mock.ts b/server/test/repositories/media.repository.mock.ts index b6b1e82b52..355180beb5 100644 --- a/server/test/repositories/media.repository.mock.ts +++ b/server/test/repositories/media.repository.mock.ts @@ -6,12 +6,11 @@ export const newMediaRepositoryMock = (): Mocked Promise.resolve()), writeExif: vitest.fn().mockImplementation(() => Promise.resolve()), - copyTagGroup: vitest.fn().mockImplementation(() => Promise.resolve()), + writeTags: vitest.fn().mockResolvedValue(true), generateThumbhash: vitest.fn().mockResolvedValue(Buffer.from('')), decodeImage: vitest.fn().mockResolvedValue({ data: Buffer.from(''), info: {} }), extract: vitest.fn().mockResolvedValue(null), probe: vitest.fn(), transcode: vitest.fn(), - getImageDimensions: vitest.fn(), }; };