Merge 33b97776ee into 5ade152bc5

server/src/repositories/media.repository.ts

@@ -39,6 +39,7 @@ type ProgressEvent = {
 export type ExtractResult = {
   buffer: Buffer;
   format: RawExtractedFormat;
+  dimensions: ImageDimensions;
 };
 
 @Injectable()
@@ -55,28 +56,28 @@ export class MediaRepository {
   async extract(input: string): Promise<ExtractResult | null> {
     try {
       const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw2', input);
-      return { buffer, format: RawExtractedFormat.Jpeg };
+      return { buffer, format: RawExtractedFormat.Jpeg, dimensions: await this.getImageDimensions(buffer) };
     } catch (error: any) {
       this.logger.debug(`Could not extract JpgFromRaw2 buffer from image, trying JPEG from RAW next: ${error}`);
     }
 
     try {
       const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw', input);
-      return { buffer, format: RawExtractedFormat.Jpeg };
+      return { buffer, format: RawExtractedFormat.Jpeg, dimensions: await this.getImageDimensions(buffer) };
     } catch (error: any) {
       this.logger.debug(`Could not extract JPEG buffer from image, trying PreviewJXL next: ${error}`);
     }
 
     try {
       const buffer = await exiftool.extractBinaryTagToBuffer('PreviewJXL', input);
-      return { buffer, format: RawExtractedFormat.Jxl };
+      return { buffer, format: RawExtractedFormat.Jxl, dimensions: await this.getImageDimensions(buffer) };
     } catch (error: any) {
       this.logger.debug(`Could not extract PreviewJXL buffer from image, trying PreviewImage next: ${error}`);
     }
 
     try {
       const buffer = await exiftool.extractBinaryTagToBuffer('PreviewImage', input);
-      return { buffer, format: RawExtractedFormat.Jpeg };
+      return { buffer, format: RawExtractedFormat.Jpeg, dimensions: await this.getImageDimensions(buffer) };
     } catch (error: any) {
       this.logger.debug(`Could not extract preview buffer from image: ${error}`);
       return null;
@@ -121,19 +122,15 @@ export class MediaRepository {
     }
   }
 
-  async copyTagGroup(tagGroup: string, source: string, target: string): Promise<boolean> {
+  async writeTags(tags: WriteTags, output: string): Promise<boolean> {
     try {
-      await exiftool.write(
+      await exiftool.write(output, tags, {
-        target,
+        ignoreMinorErrors: true,
-        {},
+        writeArgs: ['-overwrite_original'],
-        {
+      });
-          ignoreMinorErrors: true,
-          writeArgs: ['-TagsFromFile', source, `-${tagGroup}:all>${tagGroup}:all`, '-overwrite_original'],
-        },
-      );
       return true;
     } catch (error: any) {
-      this.logger.warn(`Could not copy tag data to image: ${error.message}`);
+      this.logger.warn(`Could not write tags to image: ${error.message}`);
       return false;
     }
   }
@@ -273,7 +270,7 @@ export class MediaRepository {
     });
   }
 
-  async getImageDimensions(input: string | Buffer): Promise<ImageDimensions> {
+  private async getImageDimensions(input: string | Buffer): Promise<ImageDimensions> {
     const { width = 0, height = 0 } = await sharp(input).metadata();
     return { width, height };
   }

server/src/repositories/metadata.repository.ts

@@ -72,6 +72,21 @@ export interface ImmichTags extends Omit<Tags, TagsWithWrongTypes> {
 
   AndroidMake?: string;
   AndroidModel?: string;
+
+  UsePanoramaViewer?: boolean;
+  ProjectionType?: string;
+  PoseHeadingDegrees?: number;
+  PosePitchDegrees?: number;
+  PoseRollDegrees?: number;
+  InitialViewHeadingDegrees?: number;
+  InitialViewPitchDegrees?: number;
+  InitialViewRollDegrees?: number;
+  CroppedAreaImageWidthPixels?: number;
+  CroppedAreaImageHeightPixels?: number;
+  FullPanoWidthPixels?: number;
+  FullPanoHeightPixels?: number;
+  CroppedAreaLeftPixels?: number;
+  CroppedAreaTopPixels?: number;
 }
 
 @Injectable()

server/src/services/media.service.spec.ts

@@ -599,8 +599,11 @@ describe(MediaService.name, () => {
     });
 
     it('should extract embedded image if enabled and available', async () => {
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+        buffer: extractedBuffer,
+        format: RawExtractedFormat.Jpeg,
+        dimensions: { width: 3840, height: 2160 },
+      });
       mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } });
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng);
 
@@ -615,8 +618,11 @@ describe(MediaService.name, () => {
     });
 
     it('should resize original image if embedded image is too small', async () => {
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 1000, height: 1000 });
+        buffer: extractedBuffer,
+        format: RawExtractedFormat.Jpeg,
+        dimensions: { width: 1000, height: 1000 },
+      });
       mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } });
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng);
 
@@ -641,7 +647,6 @@ describe(MediaService.name, () => {
         processInvalidImages: false,
         size: 1440,
       });
-      expect(mocks.media.getImageDimensions).not.toHaveBeenCalled();
     });
 
     it('should resize original image if embedded image extraction is not enabled', async () => {
@@ -657,7 +662,6 @@ describe(MediaService.name, () => {
         processInvalidImages: false,
         size: 1440,
       });
-      expect(mocks.media.getImageDimensions).not.toHaveBeenCalled();
     });
 
     it('should process invalid images if enabled', async () => {
@@ -691,7 +695,6 @@ describe(MediaService.name, () => {
         expect.objectContaining({ processInvalidImages: false }),
       );
 
-      expect(mocks.media.getImageDimensions).not.toHaveBeenCalled();
       vi.unstubAllEnvs();
     });
 
@@ -699,8 +702,11 @@ describe(MediaService.name, () => {
       mocks.systemMetadata.get.mockResolvedValue({
         image: { fullsize: { enabled: true, format: ImageFormat.Webp }, extractEmbedded: true },
       });
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+        buffer: extractedBuffer,
+        format: RawExtractedFormat.Jpeg,
+        dimensions: { width: 3840, height: 2160 },
+      });
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng);
 
       await sut.handleGenerateThumbnails({ id: assetStub.image.id });
@@ -731,8 +737,11 @@ describe(MediaService.name, () => {
       mocks.systemMetadata.get.mockResolvedValue({
         image: { fullsize: { enabled: true, format: ImageFormat.Webp }, extractEmbedded: true },
       });
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jxl });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+        buffer: extractedBuffer,
+        format: RawExtractedFormat.Jxl,
+        dimensions: { width: 3840, height: 2160 },
+      });
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng);
 
       await sut.handleGenerateThumbnails({ id: assetStub.image.id });
@@ -771,8 +780,11 @@ describe(MediaService.name, () => {
 
     it('should generate full-size preview directly from RAW images when extractEmbedded is false', async () => {
       mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true }, extractEmbedded: false } });
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+        buffer: extractedBuffer,
+        format: RawExtractedFormat.Jpeg,
+        dimensions: { width: 3840, height: 2160 },
+      });
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageDng);
 
       await sut.handleGenerateThumbnails({ id: assetStub.image.id });
@@ -811,8 +823,11 @@ describe(MediaService.name, () => {
 
     it('should generate full-size preview from non-web-friendly images', async () => {
       mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+        buffer: extractedBuffer,
+        format: RawExtractedFormat.Jpeg,
+        dimensions: { width: 3840, height: 2160 },
+      });
       // HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari.
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageHif);
 
@@ -840,8 +855,11 @@ describe(MediaService.name, () => {
 
     it('should skip generating full-size preview for web-friendly images', async () => {
       mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+        buffer: extractedBuffer,
+        format: RawExtractedFormat.Jpeg,
+        dimensions: { width: 3840, height: 2160 },
+      });
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.image);
 
       await sut.handleGenerateThumbnails({ id: assetStub.image.id });
@@ -863,9 +881,17 @@ describe(MediaService.name, () => {
 
     it('should always generate full-size preview from non-web-friendly panoramas', async () => {
       mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+        buffer: extractedBuffer,
-      mocks.media.copyTagGroup.mockResolvedValue(true);
+        format: RawExtractedFormat.Jpeg,
+        dimensions: { width: 3840, height: 2160 },
+      });
+      mocks.metadata.readTags.mockResolvedValue({
+        ProjectionType: 'equirectangular',
+        PoseHeadingDegrees: 127,
+        FullPanoWidthPixels: 3840,
+        FullPanoHeightPixels: 2160,
+      });
 
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.panoramaTif);
 
@@ -892,10 +918,14 @@ describe(MediaService.name, () => {
         expect.any(String),
       );
 
-      expect(mocks.media.copyTagGroup).toHaveBeenCalledTimes(2);
+      expect(mocks.media.writeTags).toHaveBeenCalledTimes(2);
-      expect(mocks.media.copyTagGroup).toHaveBeenCalledWith(
+      expect(mocks.media.writeTags).toHaveBeenCalledWith(
-        'XMP-GPano',
+        {
-        assetStub.panoramaTif.originalPath,
+          ProjectionType: 'equirectangular',
+          PoseHeadingDegrees: 127,
+          FullPanoWidthPixels: 2560,
+          FullPanoHeightPixels: 1440,
+        },
         expect.any(String),
       );
     });
@@ -904,8 +934,11 @@ describe(MediaService.name, () => {
       mocks.systemMetadata.get.mockResolvedValue({
         image: { fullsize: { enabled: true, format: ImageFormat.Webp, quality: 90 } },
       });
-      mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({
-      mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });
+        buffer: extractedBuffer,
+        format: RawExtractedFormat.Jpeg,
+        dimensions: { width: 3840, height: 2160 },
+      });
       // HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari.
       mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.imageHif);
 
@@ -1190,9 +1223,8 @@ describe(MediaService.name, () => {
       const extracted = Buffer.from('');
       const data = Buffer.from('');
       const info = { width: 2160, height: 3840 } as OutputInfo;
-      mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg, dimensions: info });
       mocks.media.decodeImage.mockResolvedValue({ data, info });
-      mocks.media.getImageDimensions.mockResolvedValue(info);
 
       await expect(sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id })).resolves.toBe(
         JobStatus.Success,
@@ -1268,8 +1300,7 @@ describe(MediaService.name, () => {
       const data = Buffer.from('');
       const info = { width: 1000, height: 1000 } as OutputInfo;
       mocks.media.decodeImage.mockResolvedValue({ data, info });
-      mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg });
+      mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg, dimensions: info });
-      mocks.media.getImageDimensions.mockResolvedValue(info);
 
       await expect(sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id })).resolves.toBe(
         JobStatus.Success,

server/src/services/media.service.ts

@@ -47,6 +47,26 @@ interface UpsertFileOptions {
   path: string;
 }
 
+const PANORAMA_CONSTANTS = [
+  'UsePanoramaViewer',
+  'ProjectionType',
+  'PoseHeadingDegrees',
+  'PosePitchDegrees',
+  'PoseRollDegrees',
+  'InitialViewHeadingDegrees',
+  'InitialViewPitchDegrees',
+  'InitialViewRollDegrees',
+] as const;
+
+const PANORAMA_SCALABLES = [
+  'CroppedAreaImageWidthPixels',
+  'CroppedAreaImageHeightPixels',
+  'FullPanoWidthPixels',
+  'FullPanoHeightPixels',
+  'CroppedAreaLeftPixels',
+  'CroppedAreaTopPixels',
+] as const;
+
 @Injectable()
 export class MediaService extends BaseService {
   videoInterfaces: VideoInterfaces = { dri: [], mali: false };
@@ -237,7 +257,7 @@ export class MediaService extends BaseService {
 
   private async extractImage(originalPath: string, minSize: number) {
     let extracted = await this.mediaRepository.extract(originalPath);
-    if (extracted && !(await this.shouldUseExtractedImage(extracted.buffer, minSize))) {
+    if (extracted && !this.shouldUseExtractedImage(extracted.dimensions, minSize)) {
       extracted = null;
     }
 
@@ -295,14 +315,21 @@ export class MediaService extends BaseService {
     ];
 
     let fullsizePath: string | undefined;
+    let fullsizeSize: number | undefined;
+    const originalSize =
+      asset.exifInfo.exifImageWidth && asset.exifInfo.exifImageHeight
+        ? Math.min(asset.exifInfo.exifImageWidth, asset.exifInfo.exifImageHeight)
+        : undefined;
 
     if (convertFullsize) {
       // convert a new fullsize image from the same source as the thumbnail
       fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FullSize, image.fullsize.format);
+      fullsizeSize = originalSize;
       const fullsizeOptions = { format: image.fullsize.format, quality: image.fullsize.quality, ...thumbnailOptions };
       promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizePath));
     } else if (generateFullsize && extracted && extracted.format === RawExtractedFormat.Jpeg) {
       fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FullSize, extracted.format);
+      fullsizeSize = Math.min(extracted.dimensions.width, extracted.dimensions.height);
       this.storageCore.ensureFolders(fullsizePath);
 
       // Write the buffer to disk with essential EXIF data
@@ -318,19 +345,55 @@ export class MediaService extends BaseService {
 
     const outputs = await Promise.all(promises);
 
-    if (asset.exifInfo.projectionType === 'EQUIRECTANGULAR') {
+    if (asset.exifInfo.projectionType === 'EQUIRECTANGULAR' && originalSize) {
-      const promises = [
+      await this.copyPanoramaMetadataToThumbnails(
-        this.mediaRepository.copyTagGroup('XMP-GPano', asset.originalPath, previewPath),
+        asset.originalPath,
-        fullsizePath
+        originalSize,
-          ? this.mediaRepository.copyTagGroup('XMP-GPano', asset.originalPath, fullsizePath)
+        previewPath,
-          : Promise.resolve(),
+        image.preview.size,
-      ];
+        fullsizePath,
-      await Promise.all(promises);
+        fullsizeSize,
+      );
     }
 
     return { previewPath, thumbnailPath, fullsizePath, thumbhash: outputs[0] as Buffer };
   }
 
+  private async copyPanoramaMetadataToThumbnails(
+    originalPath: string,
+    originalSize: number,
+    previewPath: string,
+    previewSize: number,
+    fullsizePath?: string,
+    fullsizeSize?: number,
+  ) {
+    const originalTags = await this.metadataRepository.readTags(originalPath);
+
+    const scaleAndWriteData = async (thumbnailPath: string, scaleRatio: number) => {
+      const newTags = {} as Record<string, string | number | boolean>;
+
+      for (const key of PANORAMA_CONSTANTS) {
+        if (key in originalTags && originalTags[key]) {
+          newTags[key] = originalTags[key];
+        }
+      }
+      for (const key of PANORAMA_SCALABLES) {
+        if (key in originalTags && originalTags[key]) {
+          newTags[key] = Math.round(originalTags[key] * scaleRatio);
+        }
+      }
+
+      return this.mediaRepository.writeTags(newTags, thumbnailPath);
+    };
+
+    const promises = [
+      // preview size is min(preview size, original size) so do the same for pano pixel adjustment
+      scaleAndWriteData(previewPath, Math.min(1, previewSize / originalSize)),
+      fullsizePath && fullsizeSize ? scaleAndWriteData(fullsizePath, fullsizeSize / originalSize) : Promise.resolve(),
+    ];
+    await Promise.all(promises);
+  }
+
   @OnJob({ name: JobName.PersonGenerateThumbnail, queue: QueueName.ThumbnailGeneration })
   async handleGeneratePersonThumbnail({ id }: JobOf<JobName.PersonGenerateThumbnail>): Promise<JobStatus> {
     const { machineLearning, metadata, image } = await this.getConfig({ withCache: true });
@@ -680,8 +743,8 @@ export class MediaService extends BaseService {
     }
   }
 
-  private async shouldUseExtractedImage(extractedPathOrBuffer: string | Buffer, targetSize: number) {
+  private shouldUseExtractedImage(extractedDimensions: ImageDimensions, targetSize: number) {
-    const { width, height } = await this.mediaRepository.getImageDimensions(extractedPathOrBuffer);
+    const { width, height } = extractedDimensions;
     const extractedSize = Math.min(width, height);
     return extractedSize >= targetSize;
   }

server/test/fixtures/asset.stub.ts

@@ -892,6 +892,8 @@ export const assetStub = {
     exifInfo: {
       fileSizeInByte: 5000,
       projectionType: 'EQUIRECTANGULAR',
+      exifImageHeight: 2160,
+      exifImageWidth: 3840,
     } as Exif,
     duplicateId: null,
     isOffline: false,

server/test/repositories/media.repository.mock.ts

@@ -6,12 +6,11 @@ export const newMediaRepositoryMock = (): Mocked<RepositoryInterface<MediaReposi
   return {
     generateThumbnail: vitest.fn().mockImplementation(() => Promise.resolve()),
     writeExif: vitest.fn().mockImplementation(() => Promise.resolve()),
-    copyTagGroup: vitest.fn().mockImplementation(() => Promise.resolve()),
+    writeTags: vitest.fn().mockResolvedValue(true),
     generateThumbhash: vitest.fn().mockResolvedValue(Buffer.from('')),
     decodeImage: vitest.fn().mockResolvedValue({ data: Buffer.from(''), info: {} }),
     extract: vitest.fn().mockResolvedValue(null),
     probe: vitest.fn(),
     transcode: vitest.fn(),
-    getImageDimensions: vitest.fn(),
   };
 };