From f77f43a83d52ef18eff1a29ea2e144bfb2f1cfed Mon Sep 17 00:00:00 2001 From: izzy Date: Wed, 26 Nov 2025 15:45:58 +0000 Subject: [PATCH 01/50] stash: integrity checks --- open-api/immich-openapi-specs.json | 4 +- server/src/enum.ts | 4 + server/src/repositories/asset.repository.ts | 8 ++ server/src/services/index.ts | 2 + server/src/services/integrity.service.ts | 120 ++++++++++++++++++++ server/src/types.ts | 6 +- 6 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 server/src/services/integrity.service.ts diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 68af1438cd..80e590e63e 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -16660,7 +16660,9 @@ "VersionCheck", "OcrQueueAll", "Ocr", - "WorkflowRun" + "WorkflowRun", + "IntegrityOrphanedAndMissingFiles", + "IntegrityChecksumFiles" ], "type": "string" }, diff --git a/server/src/enum.ts b/server/src/enum.ts index 87ff282f31..f1425cf0f8 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -637,6 +637,10 @@ export enum JobName { // Workflow WorkflowRun = 'WorkflowRun', + + // Integrity + IntegrityOrphanedAndMissingFiles = 'IntegrityOrphanedAndMissingFiles', + IntegrityChecksumFiles = 'IntegrityChecksumFiles', } export enum QueueCommand { diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index d3d9ada80f..226d021745 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -382,6 +382,14 @@ export class AssetRepository { return items.map((asset) => asset.deviceAssetId); } + async getAllAssetPaths() { + return this.db.selectFrom('asset').select(['originalPath', 'encodedVideoPath']).stream(); + } + + async getAllAssetFilePaths() { + return this.db.selectFrom('asset_file').select(['path']).stream(); + } + @GenerateSql({ params: [DummyValue.UUID] }) async getLivePhotoCount(motionId: string): Promise { const [{ count }] = await this.db diff --git a/server/src/services/index.ts b/server/src/services/index.ts index eeb8424048..42bf467fd0 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -12,6 +12,7 @@ import { CliService } from 'src/services/cli.service'; import { DatabaseService } from 'src/services/database.service'; import { DownloadService } from 'src/services/download.service'; import { DuplicateService } from 'src/services/duplicate.service'; +import { IntegrityService } from 'src/services/integrity.service'; import { JobService } from 'src/services/job.service'; import { LibraryService } from 'src/services/library.service'; import { MaintenanceService } from 'src/services/maintenance.service'; @@ -62,6 +63,7 @@ export const services = [ DatabaseService, DownloadService, DuplicateService, + IntegrityService, JobService, LibraryService, MaintenanceService, diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts new file mode 100644 index 0000000000..70c525dfc7 --- /dev/null +++ b/server/src/services/integrity.service.ts @@ -0,0 +1,120 @@ +import { Injectable } from '@nestjs/common'; +import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; +import { StorageCore } from 'src/cores/storage.core'; +import { OnEvent, OnJob } from 'src/decorators'; +import { ImmichWorker, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum'; +import { ArgOf } from 'src/repositories/event.repository'; +import { BaseService } from 'src/services/base.service'; + +@Injectable() +export class IntegrityService extends BaseService { + // private backupLock = false; + + @OnEvent({ name: 'ConfigInit', workers: [ImmichWorker.Microservices] }) + async onConfigInit({ + newConfig: { + backup: { database }, + }, + }: ArgOf<'ConfigInit'>) { + // this.backupLock = await this.databaseRepository.tryLock(DatabaseLock.BackupDatabase); + // if (this.backupLock) { + // this.cronRepository.create({ + // name: 'backupDatabase', + // expression: database.cronExpression, + // onTick: () => handlePromiseError(this.jobRepository.queue({ name: JobName.DatabaseBackup }), this.logger), + // start: database.enabled, + // }); + // } + setTimeout(() => { + this.jobRepository.queue({ + name: JobName.IntegrityOrphanedAndMissingFiles, + data: {}, + }); + }, 1000); + } + + @OnEvent({ name: 'ConfigUpdate', server: true }) + async onConfigUpdate({ newConfig: { backup } }: ArgOf<'ConfigUpdate'>) { + // if (!this.backupLock) { + // return; + // } + // this.cronRepository.update({ + // name: 'backupDatabase', + // expression: backup.database.cronExpression, + // start: backup.database.enabled, + // }); + } + + @OnJob({ name: JobName.IntegrityOrphanedAndMissingFiles, queue: QueueName.BackgroundTask }) + async handleOrphanedAndMissingFiles(): Promise { + // (1) Asset files + const pathsLocal = new Set(); + const pathsDb = new Set(); + + await Promise.all([ + // scan all local paths + (async () => { + const pathsOnDisk = this.storageRepository.walk({ + pathsToCrawl: [ + StorageFolder.EncodedVideo, + StorageFolder.Library, + StorageFolder.Upload, + StorageFolder.Thumbnails, + ].map((folder) => StorageCore.getBaseFolder(folder)), + includeHidden: false, + take: JOBS_LIBRARY_PAGINATION_SIZE, + }); + + for await (const pathBatch of pathsOnDisk) { + for (const path of pathBatch) { + if (!pathsDb.delete(path)) { + pathsLocal.add(path); + } + + console.info(pathsLocal.size, pathsDb.size); + } + } + })(), + // scan "asset" + (async () => { + const pathsInDb = await this.assetRepository.getAllAssetPaths(); + + for await (const { originalPath, encodedVideoPath } of pathsInDb) { + if (!pathsLocal.delete(originalPath)) { + pathsDb.add(originalPath); + } + + if (encodedVideoPath && !pathsLocal.delete(encodedVideoPath)) { + pathsDb.add(encodedVideoPath); + } + + console.info(pathsLocal.size, pathsDb.size); + } + })(), + // scan "asset_files" + (async () => { + const pathsInDb = await this.assetRepository.getAllAssetFilePaths(); + + for await (const { path } of pathsInDb) { + if (!pathsLocal.delete(path)) { + pathsDb.add(path); + } + + console.info(pathsLocal.size, pathsDb.size); + } + })(), + ]); + + console.info('Orphaned files:', pathsLocal); + console.info('Missing files:', pathsDb); + + // profile: skipped + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.BackgroundTask }) + async handleChecksumFiles(): Promise { + // todo + return JobStatus.Success; + } +} diff --git a/server/src/types.ts b/server/src/types.ts index 848d19177d..889051f580 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -391,7 +391,11 @@ export type JobItem = | { name: JobName.Ocr; data: IEntityJob } // Workflow - | { name: JobName.WorkflowRun; data: IWorkflowJob }; + | { name: JobName.WorkflowRun; data: IWorkflowJob } + + // Integrity + | { name: JobName.IntegrityOrphanedAndMissingFiles; data: IBaseJob } + | { name: JobName.IntegrityChecksumFiles; data: IBaseJob }; export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number]; From 4a7120cdeb7e3c57627644625b3bbd4f78f5ce9e Mon Sep 17 00:00:00 2001 From: izzy Date: Wed, 26 Nov 2025 17:36:28 +0000 Subject: [PATCH 02/50] refactor: batched integrity checks --- open-api/immich-openapi-specs.json | 5 +- server/src/enum.ts | 5 +- .../src/repositories/asset-job.repository.ts | 24 +++ server/src/services/integrity.service.ts | 191 +++++++++++++----- server/src/types.ts | 14 +- 5 files changed, 184 insertions(+), 55 deletions(-) diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 80e590e63e..cc311e9335 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -16661,7 +16661,10 @@ "OcrQueueAll", "Ocr", "WorkflowRun", - "IntegrityOrphanedAndMissingFiles", + "IntegrityOrphanedFilesQueueAll", + "IntegrityOrphanedFiles", + "IntegrityMissingFilesQueueAll", + "IntegrityMissingFiles", "IntegrityChecksumFiles" ], "type": "string" diff --git a/server/src/enum.ts b/server/src/enum.ts index f1425cf0f8..58fe0fb3a7 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -639,7 +639,10 @@ export enum JobName { WorkflowRun = 'WorkflowRun', // Integrity - IntegrityOrphanedAndMissingFiles = 'IntegrityOrphanedAndMissingFiles', + IntegrityOrphanedFilesQueueAll = 'IntegrityOrphanedFilesQueueAll', + IntegrityOrphanedFiles = 'IntegrityOrphanedFiles', + IntegrityMissingFilesQueueAll = 'IntegrityMissingFilesQueueAll', + IntegrityMissingFiles = 'IntegrityMissingFiles', IntegrityChecksumFiles = 'IntegrityChecksumFiles', } diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 8d54e93c87..d7e5c1fdb8 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -254,6 +254,30 @@ export class AssetJobRepository { .executeTakeFirst(); } + @GenerateSql({ params: [DummyValue.STRING] }) + getAssetPathsByPaths(paths: string[]) { + return this.db + .selectFrom('asset') + .select(['originalPath', 'encodedVideoPath']) + .where((eb) => eb.or([eb('originalPath', 'in', paths), eb('encodedVideoPath', 'in', paths)])) + .execute(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getAssetFilePathsByPaths(paths: string[]) { + return this.db.selectFrom('asset_file').select(['path']).where('path', 'in', paths).execute(); + } + + @GenerateSql({ params: [], stream: true }) + streamAssetPaths() { + return this.db.selectFrom('asset').select(['originalPath', 'encodedVideoPath']).stream(); + } + + @GenerateSql({ params: [], stream: true }) + streamAssetFilePaths() { + return this.db.selectFrom('asset_file').select(['path']).stream(); + } + @GenerateSql({ params: [], stream: true }) streamForVideoConversion(force?: boolean) { return this.db diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index 70c525dfc7..49900cd1f2 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -1,10 +1,12 @@ import { Injectable } from '@nestjs/common'; +import { stat } from 'node:fs/promises'; import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent, OnJob } from 'src/decorators'; import { ImmichWorker, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; +import { IIntegrityMissingFilesJob, IIntegrityOrphanedFilesJob } from 'src/types'; @Injectable() export class IntegrityService extends BaseService { @@ -27,7 +29,12 @@ export class IntegrityService extends BaseService { // } setTimeout(() => { this.jobRepository.queue({ - name: JobName.IntegrityOrphanedAndMissingFiles, + name: JobName.IntegrityOrphanedFilesQueueAll, + data: {}, + }); + + this.jobRepository.queue({ + name: JobName.IntegrityMissingFilesQueueAll, data: {}, }); }, 1000); @@ -45,70 +52,150 @@ export class IntegrityService extends BaseService { // }); } - @OnJob({ name: JobName.IntegrityOrphanedAndMissingFiles, queue: QueueName.BackgroundTask }) - async handleOrphanedAndMissingFiles(): Promise { - // (1) Asset files - const pathsLocal = new Set(); - const pathsDb = new Set(); + @OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.BackgroundTask }) + async handleOrphanedFilesQueueAll(): Promise { + this.logger.log(`Scanning for orphaned files...`); - await Promise.all([ - // scan all local paths - (async () => { - const pathsOnDisk = this.storageRepository.walk({ - pathsToCrawl: [ - StorageFolder.EncodedVideo, - StorageFolder.Library, - StorageFolder.Upload, - StorageFolder.Thumbnails, - ].map((folder) => StorageCore.getBaseFolder(folder)), - includeHidden: false, - take: JOBS_LIBRARY_PAGINATION_SIZE, - }); + const assetPaths = this.storageRepository.walk({ + pathsToCrawl: [StorageFolder.EncodedVideo, StorageFolder.Library, StorageFolder.Upload].map((folder) => + StorageCore.getBaseFolder(folder), + ), + includeHidden: false, + take: JOBS_LIBRARY_PAGINATION_SIZE, + }); - for await (const pathBatch of pathsOnDisk) { - for (const path of pathBatch) { - if (!pathsDb.delete(path)) { - pathsLocal.add(path); - } + const assetFilePaths = this.storageRepository.walk({ + pathsToCrawl: [StorageCore.getBaseFolder(StorageFolder.Thumbnails)], + includeHidden: false, + take: JOBS_LIBRARY_PAGINATION_SIZE, + }); - console.info(pathsLocal.size, pathsDb.size); - } + async function* paths() { + for await (const batch of assetPaths) { + yield ['asset', batch] as const; + } + + for await (const batch of assetFilePaths) { + yield ['asset_file', batch] as const; + } + } + + let total = 0; + for await (const [batchType, batchPaths] of paths()) { + await this.jobRepository.queue({ + name: JobName.IntegrityOrphanedFiles, + data: { + type: batchType, + paths: batchPaths, + }, + }); + + const count = batchPaths.length; + total += count; + + this.logger.log(`Queued orphan check of ${count} file(s) (${total} so far)`); + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityOrphanedFiles, queue: QueueName.BackgroundTask }) + async handleOrphanedFiles({ type, paths }: IIntegrityOrphanedFilesJob): Promise { + this.logger.log(`Processing batch of ${paths.length} files to check if they are orphaned.`); + + const orphanedFiles = new Set(paths); + if (type === 'asset') { + const assets = await this.assetJobRepository.getAssetPathsByPaths(paths); + for (const { originalPath, encodedVideoPath } of assets) { + orphanedFiles.delete(originalPath); + + if (encodedVideoPath) { + orphanedFiles.delete(encodedVideoPath); } - })(), - // scan "asset" - (async () => { - const pathsInDb = await this.assetRepository.getAllAssetPaths(); + } + } else { + const assets = await this.assetJobRepository.getAssetFilePathsByPaths(paths); + for (const { path } of assets) { + orphanedFiles.delete(path); + } + } - for await (const { originalPath, encodedVideoPath } of pathsInDb) { - if (!pathsLocal.delete(originalPath)) { - pathsDb.add(originalPath); - } + // do something with orphanedFiles + console.info(orphanedFiles); - if (encodedVideoPath && !pathsLocal.delete(encodedVideoPath)) { - pathsDb.add(encodedVideoPath); - } + this.logger.log(`Processed ${paths.length} and found ${orphanedFiles.size} orphaned file(s).`); + return JobStatus.Success; + } - console.info(pathsLocal.size, pathsDb.size); + @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.BackgroundTask }) + async handleMissingFilesQueueAll(): Promise { + const assetPaths = this.assetJobRepository.streamAssetPaths(); + const assetFilePaths = this.assetJobRepository.streamAssetFilePaths(); + + async function* paths() { + for await (const { originalPath, encodedVideoPath } of assetPaths) { + yield originalPath; + + if (encodedVideoPath) { + yield encodedVideoPath; } - })(), - // scan "asset_files" - (async () => { - const pathsInDb = await this.assetRepository.getAllAssetFilePaths(); + } - for await (const { path } of pathsInDb) { - if (!pathsLocal.delete(path)) { - pathsDb.add(path); - } + for await (const { path } of assetFilePaths) { + yield path; + } + } - console.info(pathsLocal.size, pathsDb.size); + async function* chunk(generator: AsyncGenerator, n: number) { + let chunk: T[] = []; + for await (const item of generator) { + chunk.push(item); + + if (chunk.length === n) { + yield chunk; + chunk = []; } - })(), - ]); + } - console.info('Orphaned files:', pathsLocal); - console.info('Missing files:', pathsDb); + if (chunk.length) { + yield chunk; + } + } - // profile: skipped + let total = 0; + for await (const batchPaths of chunk(paths(), JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityMissingFiles, + data: { + paths: batchPaths, + }, + }); + + total += batchPaths.length; + this.logger.log(`Queued missing check of ${batchPaths.length} file(s) (${total} so far)`); + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityMissingFiles, queue: QueueName.BackgroundTask }) + async handleMissingFiles({ paths }: IIntegrityMissingFilesJob): Promise { + this.logger.log(`Processing batch of ${paths.length} files to check if they are missing.`); + + const result = await Promise.all( + paths.map((path) => + stat(path) + .then(() => void 0) + .catch(() => path), + ), + ); + + const missingFiles = result.filter((path) => path); + + // do something with missingFiles + console.info(missingFiles); + + this.logger.log(`Processed ${paths.length} and found ${missingFiles.length} missing file(s).`); return JobStatus.Success; } diff --git a/server/src/types.ts b/server/src/types.ts index 889051f580..ebc05946a3 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -282,6 +282,15 @@ export interface IWorkflowJob { event: WorkflowData[T]; } +export interface IIntegrityOrphanedFilesJob { + type: 'asset' | 'asset_file'; + paths: string[]; +} + +export interface IIntegrityMissingFilesJob { + paths: string[]; +} + export interface JobCounts { active: number; completed: number; @@ -394,7 +403,10 @@ export type JobItem = | { name: JobName.WorkflowRun; data: IWorkflowJob } // Integrity - | { name: JobName.IntegrityOrphanedAndMissingFiles; data: IBaseJob } + | { name: JobName.IntegrityOrphanedFilesQueueAll; data: IBaseJob } + | { name: JobName.IntegrityOrphanedFiles; data: IIntegrityOrphanedFilesJob } + | { name: JobName.IntegrityMissingFilesQueueAll; data: IBaseJob } + | { name: JobName.IntegrityMissingFiles; data: IIntegrityMissingFilesJob } | { name: JobName.IntegrityChecksumFiles; data: IBaseJob }; export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number]; From 341421045095c5244f1b87da09d17b5df6ebc59a Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 12:00:35 +0000 Subject: [PATCH 03/50] feat: checksum job --- server/src/enum.ts | 1 + .../src/repositories/asset-job.repository.ts | 19 +++ server/src/services/integrity.service.ts | 110 ++++++++++++++++-- server/src/types.ts | 1 + 4 files changed, 122 insertions(+), 9 deletions(-) diff --git a/server/src/enum.ts b/server/src/enum.ts index 58fe0fb3a7..9c8c49c1e8 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -301,6 +301,7 @@ export enum SystemMetadataKey { SystemFlags = 'system-flags', VersionCheckState = 'version-check-state', License = 'license', + IntegrityChecksumCheckpoint = 'integrity-checksum-checkpoint', } export enum UserMetadataKey { diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index d7e5c1fdb8..4bf68b708a 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -268,6 +268,14 @@ export class AssetJobRepository { return this.db.selectFrom('asset_file').select(['path']).where('path', 'in', paths).execute(); } + @GenerateSql({ params: [] }) + getAssetCount() { + return this.db + .selectFrom('asset') + .select((eb) => eb.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + } + @GenerateSql({ params: [], stream: true }) streamAssetPaths() { return this.db.selectFrom('asset').select(['originalPath', 'encodedVideoPath']).stream(); @@ -278,6 +286,17 @@ export class AssetJobRepository { return this.db.selectFrom('asset_file').select(['path']).stream(); } + @GenerateSql({ params: [DummyValue.DATE, DummyValue.DATE], stream: true }) + streamAssetChecksums(startMarker?: Date, endMarker?: Date) { + return this.db + .selectFrom('asset') + .select(['originalPath', 'checksum', 'createdAt']) + .$if(startMarker !== undefined, (qb) => qb.where('createdAt', '>=', startMarker!)) + .$if(endMarker !== undefined, (qb) => qb.where('createdAt', '<=', endMarker!)) + .orderBy('createdAt', 'asc') + .stream(); + } + @GenerateSql({ params: [], stream: true }) streamForVideoConversion(force?: boolean) { return this.db diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index 49900cd1f2..0347a06edf 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -1,9 +1,13 @@ import { Injectable } from '@nestjs/common'; +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; import { stat } from 'node:fs/promises'; +import { Writable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent, OnJob } from 'src/decorators'; -import { ImmichWorker, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum'; +import { ImmichWorker, JobName, JobStatus, QueueName, StorageFolder, SystemMetadataKey } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; import { IIntegrityMissingFilesJob, IIntegrityOrphanedFilesJob } from 'src/types'; @@ -28,13 +32,18 @@ export class IntegrityService extends BaseService { // }); // } setTimeout(() => { - this.jobRepository.queue({ - name: JobName.IntegrityOrphanedFilesQueueAll, - data: {}, - }); + // this.jobRepository.queue({ + // name: JobName.IntegrityOrphanedFilesQueueAll, + // data: {}, + // }); + + // this.jobRepository.queue({ + // name: JobName.IntegrityMissingFilesQueueAll, + // data: {}, + // }); this.jobRepository.queue({ - name: JobName.IntegrityMissingFilesQueueAll, + name: JobName.IntegrityChecksumFiles, data: {}, }); }, 1000); @@ -120,7 +129,7 @@ export class IntegrityService extends BaseService { } } - // do something with orphanedFiles + // todo: do something with orphanedFiles console.info(orphanedFiles); this.logger.log(`Processed ${paths.length} and found ${orphanedFiles.size} orphaned file(s).`); @@ -129,6 +138,8 @@ export class IntegrityService extends BaseService { @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.BackgroundTask }) async handleMissingFilesQueueAll(): Promise { + this.logger.log(`Scanning for missing files...`); + const assetPaths = this.assetJobRepository.streamAssetPaths(); const assetFilePaths = this.assetJobRepository.streamAssetFilePaths(); @@ -192,7 +203,7 @@ export class IntegrityService extends BaseService { const missingFiles = result.filter((path) => path); - // do something with missingFiles + // todo: do something with missingFiles console.info(missingFiles); this.logger.log(`Processed ${paths.length} and found ${missingFiles.length} missing file(s).`); @@ -201,7 +212,88 @@ export class IntegrityService extends BaseService { @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.BackgroundTask }) async handleChecksumFiles(): Promise { - // todo + const timeLimit = 60 * 60 * 1000; // 1000; + const percentageLimit = 1.0; // 0.25; + + this.logger.log( + `Checking file checksums... (will run for up to ${(timeLimit / (60 * 60 * 1000)).toFixed(2)} hours or until ${(percentageLimit * 100).toFixed(2)}% of assets are processed)`, + ); + + let processed = 0; + const startedAt = Date.now(); + const { count } = await this.assetJobRepository.getAssetCount(); + const checkpoint = await this.systemMetadataRepository.get(SystemMetadataKey.IntegrityChecksumCheckpoint); + + let startMarker: Date | undefined = checkpoint?.date ? new Date(checkpoint.date) : undefined; + let endMarker: Date | undefined; // todo + + const printStats = () => { + const averageTime = ((Date.now() - startedAt) / processed).toFixed(2); + const completionProgress = ((processed / count) * 100).toFixed(2); + + this.logger.log( + `Processed ${processed} files so far... (avg. ${averageTime} ms/asset, ${completionProgress}% of all assets)`, + ); + }; + + let lastCreatedAt: Date | undefined; + + finishEarly: do { + this.logger.log( + `Processing assets in range [${startMarker?.toISOString() ?? 'beginning'}, ${endMarker?.toISOString() ?? 'end'}]`, + ); + + const assets = this.assetJobRepository.streamAssetChecksums(startMarker, endMarker); + endMarker = startMarker; + startMarker = undefined; + + for await (const { originalPath, checksum, createdAt } of assets) { + try { + const hash = createHash('sha1'); + + await pipeline([ + createReadStream(originalPath), + new Writable({ + write(chunk, _encoding, callback) { + hash.update(chunk); + callback(); + }, + }), + ]); + + if (!checksum.equals(hash.digest())) { + throw new Error('File failed checksum'); + } + } catch (error) { + this.logger.warn('Failed to process a file: ' + error); + // todo: do something with originalPath + } + + processed++; + if (processed % 100 === 0) { + printStats(); + } + + if (Date.now() > startedAt + timeLimit || processed > count * percentageLimit) { + this.logger.log('Reached stop criteria.'); + lastCreatedAt = createdAt; + break finishEarly; + } + } + } while (endMarker); + + this.systemMetadataRepository.set(SystemMetadataKey.IntegrityChecksumCheckpoint, { + date: lastCreatedAt?.toISOString(), + }); + + printStats(); + + if (lastCreatedAt) { + this.logger.log(`Finished checksum job, will continue from ${lastCreatedAt.toISOString()}.`); + } else { + this.logger.log(`Finished checksum job, covered all assets.`); + } + return JobStatus.Success; } } diff --git a/server/src/types.ts b/server/src/types.ts index ebc05946a3..183bdfc750 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -522,6 +522,7 @@ export interface SystemMetadata extends Record; [SystemMetadataKey.VersionCheckState]: VersionCheckMetadata; [SystemMetadataKey.MemoriesState]: MemoriesState; + [SystemMetadataKey.IntegrityChecksumCheckpoint]: { date?: string }; } export interface UserPreferences { From 15503b150ac79844f6bc0aadbf0ac21e81401f1e Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 12:01:26 +0000 Subject: [PATCH 04/50] chore: open api --- mobile/openapi/lib/model/job_name.dart | 15 +++++++++++++++ open-api/typescript-sdk/src/fetch-client.ts | 7 ++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/mobile/openapi/lib/model/job_name.dart b/mobile/openapi/lib/model/job_name.dart index 038a17a8e6..bd2e599529 100644 --- a/mobile/openapi/lib/model/job_name.dart +++ b/mobile/openapi/lib/model/job_name.dart @@ -78,6 +78,11 @@ class JobName { static const ocrQueueAll = JobName._(r'OcrQueueAll'); static const ocr = JobName._(r'Ocr'); static const workflowRun = JobName._(r'WorkflowRun'); + static const integrityOrphanedFilesQueueAll = JobName._(r'IntegrityOrphanedFilesQueueAll'); + static const integrityOrphanedFiles = JobName._(r'IntegrityOrphanedFiles'); + static const integrityMissingFilesQueueAll = JobName._(r'IntegrityMissingFilesQueueAll'); + static const integrityMissingFiles = JobName._(r'IntegrityMissingFiles'); + static const integrityChecksumFiles = JobName._(r'IntegrityChecksumFiles'); /// List of all possible values in this [enum][JobName]. static const values = [ @@ -136,6 +141,11 @@ class JobName { ocrQueueAll, ocr, workflowRun, + integrityOrphanedFilesQueueAll, + integrityOrphanedFiles, + integrityMissingFilesQueueAll, + integrityMissingFiles, + integrityChecksumFiles, ]; static JobName? fromJson(dynamic value) => JobNameTypeTransformer().decode(value); @@ -229,6 +239,11 @@ class JobNameTypeTransformer { case r'OcrQueueAll': return JobName.ocrQueueAll; case r'Ocr': return JobName.ocr; case r'WorkflowRun': return JobName.workflowRun; + case r'IntegrityOrphanedFilesQueueAll': return JobName.integrityOrphanedFilesQueueAll; + case r'IntegrityOrphanedFiles': return JobName.integrityOrphanedFiles; + case r'IntegrityMissingFilesQueueAll': return JobName.integrityMissingFilesQueueAll; + case r'IntegrityMissingFiles': return JobName.integrityMissingFiles; + case r'IntegrityChecksumFiles': return JobName.integrityChecksumFiles; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index bbcc2311b6..b3242b5d29 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -5486,7 +5486,12 @@ export enum JobName { VersionCheck = "VersionCheck", OcrQueueAll = "OcrQueueAll", Ocr = "Ocr", - WorkflowRun = "WorkflowRun" + WorkflowRun = "WorkflowRun", + IntegrityOrphanedFilesQueueAll = "IntegrityOrphanedFilesQueueAll", + IntegrityOrphanedFiles = "IntegrityOrphanedFiles", + IntegrityMissingFilesQueueAll = "IntegrityMissingFilesQueueAll", + IntegrityMissingFiles = "IntegrityMissingFiles", + IntegrityChecksumFiles = "IntegrityChecksumFiles" } export enum SearchSuggestionType { Country = "country", From 1e941f3f8891971bcd628f075f58c24d9622e734 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 12:53:04 +0000 Subject: [PATCH 05/50] feat: write integrity report to database --- server/src/enum.ts | 6 +++ server/src/repositories/index.ts | 2 + .../integrity-report.repository.ts | 19 +++++++ server/src/schema/index.ts | 4 ++ ...764246650423-CreateIntegrityReportTable.ts | 15 ++++++ .../schema/tables/integrity-report.table.ts | 21 ++++++++ server/src/services/base.service.ts | 2 + server/src/services/integrity.service.ts | 52 +++++++++++++------ 8 files changed, 105 insertions(+), 16 deletions(-) create mode 100644 server/src/repositories/integrity-report.repository.ts create mode 100644 server/src/schema/migrations/1764246650423-CreateIntegrityReportTable.ts create mode 100644 server/src/schema/tables/integrity-report.table.ts diff --git a/server/src/enum.ts b/server/src/enum.ts index 9c8c49c1e8..35fd98c53c 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -482,6 +482,12 @@ export enum CacheControl { None = 'none', } +export enum IntegrityReportType { + OrphanFile = 'orphan_file', + MissingFile = 'missing_file', + ChecksumFail = 'checksum_fail', +} + export enum ImmichEnvironment { Development = 'development', Testing = 'testing', diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index c59110d674..dc63d427a2 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -15,6 +15,7 @@ import { DownloadRepository } from 'src/repositories/download.repository'; import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityReportRepository } from 'src/repositories/integrity-report.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -68,6 +69,7 @@ export const repositories = [ DuplicateRepository, EmailRepository, EventRepository, + IntegrityReportRepository, JobRepository, LibraryRepository, LoggingRepository, diff --git a/server/src/repositories/integrity-report.repository.ts b/server/src/repositories/integrity-report.repository.ts new file mode 100644 index 0000000000..17e85e78a3 --- /dev/null +++ b/server/src/repositories/integrity-report.repository.ts @@ -0,0 +1,19 @@ +import { Injectable } from '@nestjs/common'; +import { Insertable, Kysely } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; +import { DB } from 'src/schema'; +import { IntegrityReportTable } from 'src/schema/tables/integrity-report.table'; + +@Injectable() +export class IntegrityReportRepository { + constructor(@InjectKysely() private db: Kysely) {} + + create(dto: Insertable | Insertable[]) { + return this.db + .insertInto('integrity_report') + .values(dto) + .onConflict((oc) => oc.doNothing()) + .returningAll() + .executeTakeFirst(); + } +} diff --git a/server/src/schema/index.ts b/server/src/schema/index.ts index 9e206826e6..cea30cded1 100644 --- a/server/src/schema/index.ts +++ b/server/src/schema/index.ts @@ -40,6 +40,7 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { AuditTable } from 'src/schema/tables/audit.table'; import { FaceSearchTable } from 'src/schema/tables/face-search.table'; import { GeodataPlacesTable } from 'src/schema/tables/geodata-places.table'; +import { IntegrityReportTable } from 'src/schema/tables/integrity-report.table'; import { LibraryTable } from 'src/schema/tables/library.table'; import { MemoryAssetAuditTable } from 'src/schema/tables/memory-asset-audit.table'; import { MemoryAssetTable } from 'src/schema/tables/memory-asset.table'; @@ -98,6 +99,7 @@ export class ImmichDatabase { AssetExifTable, FaceSearchTable, GeodataPlacesTable, + IntegrityReportTable, LibraryTable, MemoryTable, MemoryAuditTable, @@ -195,6 +197,8 @@ export interface DB { geodata_places: GeodataPlacesTable; + integrity_report: IntegrityReportTable; + library: LibraryTable; memory: MemoryTable; diff --git a/server/src/schema/migrations/1764246650423-CreateIntegrityReportTable.ts b/server/src/schema/migrations/1764246650423-CreateIntegrityReportTable.ts new file mode 100644 index 0000000000..a6f99ab755 --- /dev/null +++ b/server/src/schema/migrations/1764246650423-CreateIntegrityReportTable.ts @@ -0,0 +1,15 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql`CREATE TABLE "integrity_report" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "type" character varying NOT NULL, + "path" character varying NOT NULL, + CONSTRAINT "integrity_report_type_path_uq" UNIQUE ("type", "path"), + CONSTRAINT "integrity_report_pkey" PRIMARY KEY ("id") +);`.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`DROP TABLE "integrity_report";`.execute(db); +} diff --git a/server/src/schema/tables/integrity-report.table.ts b/server/src/schema/tables/integrity-report.table.ts new file mode 100644 index 0000000000..d789c534db --- /dev/null +++ b/server/src/schema/tables/integrity-report.table.ts @@ -0,0 +1,21 @@ +import { IntegrityReportType } from 'src/enum'; +import { + Column, + Generated, + PrimaryGeneratedColumn, + Table, + Unique +} from 'src/sql-tools'; + +@Table('integrity_report') +@Unique({ columns: ['type', 'path'] }) +export class IntegrityReportTable { + @PrimaryGeneratedColumn() + id!: Generated; + + @Column() + type!: IntegrityReportType; + + @Column() + path!: string; +} diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index 9c422818b3..e311a860e3 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -22,6 +22,7 @@ import { DownloadRepository } from 'src/repositories/download.repository'; import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityReportRepository } from 'src/repositories/integrity-report.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -137,6 +138,7 @@ export class BaseService { protected duplicateRepository: DuplicateRepository, protected emailRepository: EmailRepository, protected eventRepository: EventRepository, + protected integrityReportRepository: IntegrityReportRepository, protected jobRepository: JobRepository, protected libraryRepository: LibraryRepository, protected machineLearningRepository: MachineLearningRepository, diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index 0347a06edf..c4db5d49e7 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -7,7 +7,15 @@ import { pipeline } from 'node:stream/promises'; import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent, OnJob } from 'src/decorators'; -import { ImmichWorker, JobName, JobStatus, QueueName, StorageFolder, SystemMetadataKey } from 'src/enum'; +import { + ImmichWorker, + IntegrityReportType, + JobName, + JobStatus, + QueueName, + StorageFolder, + SystemMetadataKey, +} from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; import { IIntegrityMissingFilesJob, IIntegrityOrphanedFilesJob } from 'src/types'; @@ -31,16 +39,17 @@ export class IntegrityService extends BaseService { // start: database.enabled, // }); // } - setTimeout(() => { - // this.jobRepository.queue({ - // name: JobName.IntegrityOrphanedFilesQueueAll, - // data: {}, - // }); - // this.jobRepository.queue({ - // name: JobName.IntegrityMissingFilesQueueAll, - // data: {}, - // }); + setTimeout(() => { + this.jobRepository.queue({ + name: JobName.IntegrityOrphanedFilesQueueAll, + data: {}, + }); + + this.jobRepository.queue({ + name: JobName.IntegrityMissingFilesQueueAll, + data: {}, + }); this.jobRepository.queue({ name: JobName.IntegrityChecksumFiles, @@ -129,8 +138,12 @@ export class IntegrityService extends BaseService { } } - // todo: do something with orphanedFiles - console.info(orphanedFiles); + await this.integrityReportRepository.create( + [...orphanedFiles].map((path) => ({ + type: IntegrityReportType.OrphanFile, + path, + })), + ); this.logger.log(`Processed ${paths.length} and found ${orphanedFiles.size} orphaned file(s).`); return JobStatus.Success; @@ -201,10 +214,14 @@ export class IntegrityService extends BaseService { ), ); - const missingFiles = result.filter((path) => path); + const missingFiles = result.filter((path) => path) as string[]; - // todo: do something with missingFiles - console.info(missingFiles); + await this.integrityReportRepository.create( + missingFiles.map((path) => ({ + type: IntegrityReportType.MissingFile, + path, + })), + ); this.logger.log(`Processed ${paths.length} and found ${missingFiles.length} missing file(s).`); return JobStatus.Success; @@ -266,7 +283,10 @@ export class IntegrityService extends BaseService { } } catch (error) { this.logger.warn('Failed to process a file: ' + error); - // todo: do something with originalPath + await this.integrityReportRepository.create({ + path: originalPath, + type: IntegrityReportType.ChecksumFail, + }); } processed++; From 929ad529f4db2ebfc2711ed0617b3c96294f5443 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 15:13:00 +0000 Subject: [PATCH 06/50] feat: add createdAt to integrity report table refactor: rename checksum_fail to checksum_mismatched --- server/src/enum.ts | 2 +- ...ts => 1764255490085-CreateIntegrityReportTable.ts} | 1 + server/src/schema/tables/integrity-report.table.ts | 11 ++++------- 3 files changed, 6 insertions(+), 8 deletions(-) rename server/src/schema/migrations/{1764246650423-CreateIntegrityReportTable.ts => 1764255490085-CreateIntegrityReportTable.ts} (89%) diff --git a/server/src/enum.ts b/server/src/enum.ts index 35fd98c53c..145067b5e3 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -485,7 +485,7 @@ export enum CacheControl { export enum IntegrityReportType { OrphanFile = 'orphan_file', MissingFile = 'missing_file', - ChecksumFail = 'checksum_fail', + ChecksumFail = 'checksum_mismatch', } export enum ImmichEnvironment { diff --git a/server/src/schema/migrations/1764246650423-CreateIntegrityReportTable.ts b/server/src/schema/migrations/1764255490085-CreateIntegrityReportTable.ts similarity index 89% rename from server/src/schema/migrations/1764246650423-CreateIntegrityReportTable.ts rename to server/src/schema/migrations/1764255490085-CreateIntegrityReportTable.ts index a6f99ab755..a8e4c0f18f 100644 --- a/server/src/schema/migrations/1764246650423-CreateIntegrityReportTable.ts +++ b/server/src/schema/migrations/1764255490085-CreateIntegrityReportTable.ts @@ -5,6 +5,7 @@ export async function up(db: Kysely): Promise { "id" uuid NOT NULL DEFAULT uuid_generate_v4(), "type" character varying NOT NULL, "path" character varying NOT NULL, + "createdAt" timestamp with time zone NOT NULL DEFAULT now(), CONSTRAINT "integrity_report_type_path_uq" UNIQUE ("type", "path"), CONSTRAINT "integrity_report_pkey" PRIMARY KEY ("id") );`.execute(db); diff --git a/server/src/schema/tables/integrity-report.table.ts b/server/src/schema/tables/integrity-report.table.ts index d789c534db..fe983f09d6 100644 --- a/server/src/schema/tables/integrity-report.table.ts +++ b/server/src/schema/tables/integrity-report.table.ts @@ -1,11 +1,5 @@ import { IntegrityReportType } from 'src/enum'; -import { - Column, - Generated, - PrimaryGeneratedColumn, - Table, - Unique -} from 'src/sql-tools'; +import { Column, CreateDateColumn, Generated, PrimaryGeneratedColumn, Table, Timestamp, Unique } from 'src/sql-tools'; @Table('integrity_report') @Unique({ columns: ['type', 'path'] }) @@ -18,4 +12,7 @@ export class IntegrityReportTable { @Column() path!: string; + + @CreateDateColumn() + createdAt!: Generated; } From cc31b9c7f172eb2cd2d32efe3eac8025f6dc9ba0 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 15:13:19 +0000 Subject: [PATCH 07/50] feat: clean up old reports of checksum or missing files refactor: combine the stream query --- server/src/bin/migrations.ts | 8 +++ .../src/repositories/asset-job.repository.ts | 38 ++++++++--- .../integrity-report.repository.ts | 8 +++ server/src/services/integrity.service.ts | 68 ++++++++++--------- server/src/types.ts | 2 +- 5 files changed, 84 insertions(+), 40 deletions(-) diff --git a/server/src/bin/migrations.ts b/server/src/bin/migrations.ts index 588f358023..8c873c4274 100644 --- a/server/src/bin/migrations.ts +++ b/server/src/bin/migrations.ts @@ -62,6 +62,10 @@ const main = async () => { const getDatabaseClient = () => { const configRepository = new ConfigRepository(); const { database } = configRepository.getEnv(); + database.config = { + connectionType: 'url', + url: 'postgres://postgres:postgres@database:5432/immich', + }; return new Kysely(getKyselyConfig(database.config)); }; @@ -130,6 +134,10 @@ const create = (path: string, up: string[], down: string[]) => { const compare = async () => { const configRepository = new ConfigRepository(); const { database } = configRepository.getEnv(); + database.config = { + connectionType: 'url', + url: 'postgres://postgres:postgres@database:5432/immich', + }; const db = postgres(asPostgresConnectionConfig(database.config)); const source = schemaFromCode({ overrides: true, namingStrategy: 'default' }); diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 4bf68b708a..79727520b1 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -4,7 +4,7 @@ import { jsonArrayFrom } from 'kysely/helpers/postgres'; import { InjectKysely } from 'nestjs-kysely'; import { Asset, columns } from 'src/database'; import { DummyValue, GenerateSql } from 'src/decorators'; -import { AssetFileType, AssetType, AssetVisibility } from 'src/enum'; +import { AssetFileType, AssetType, AssetVisibility, IntegrityReportType } from 'src/enum'; import { DB } from 'src/schema'; import { StorageAsset } from 'src/types'; import { @@ -278,19 +278,41 @@ export class AssetJobRepository { @GenerateSql({ params: [], stream: true }) streamAssetPaths() { - return this.db.selectFrom('asset').select(['originalPath', 'encodedVideoPath']).stream(); - } - - @GenerateSql({ params: [], stream: true }) - streamAssetFilePaths() { - return this.db.selectFrom('asset_file').select(['path']).stream(); + return this.db + .selectFrom((eb) => + eb + .selectFrom('asset') + .select(['originalPath as path']) + .unionAll( + eb + .selectFrom('asset') + .select(['encodedVideoPath as path']) + .where('encodedVideoPath', 'is not', null) + .where('encodedVideoPath', '!=', '') + .$castTo<{ path: string }>(), + ) + .unionAll(eb.selectFrom('asset_file').select(['path'])) + .as('allPaths'), + ) + .leftJoin('integrity_report', (join) => + join + .onRef('integrity_report.path', '=', 'allPaths.path') + .on('integrity_report.type', '=', IntegrityReportType.OrphanFile), + ) + .select(['allPaths.path as path', 'integrity_report.path as reportId']) + .stream(); } @GenerateSql({ params: [DummyValue.DATE, DummyValue.DATE], stream: true }) streamAssetChecksums(startMarker?: Date, endMarker?: Date) { return this.db .selectFrom('asset') - .select(['originalPath', 'checksum', 'createdAt']) + .leftJoin('integrity_report', (join) => + join + .onRef('integrity_report.path', '=', 'asset.originalPath') + .on('integrity_report.type', '=', IntegrityReportType.ChecksumFail), + ) + .select(['asset.originalPath', 'asset.checksum', 'asset.createdAt', 'integrity_report.id as reportId']) .$if(startMarker !== undefined, (qb) => qb.where('createdAt', '>=', startMarker!)) .$if(endMarker !== undefined, (qb) => qb.where('createdAt', '<=', endMarker!)) .orderBy('createdAt', 'asc') diff --git a/server/src/repositories/integrity-report.repository.ts b/server/src/repositories/integrity-report.repository.ts index 17e85e78a3..25194731d6 100644 --- a/server/src/repositories/integrity-report.repository.ts +++ b/server/src/repositories/integrity-report.repository.ts @@ -16,4 +16,12 @@ export class IntegrityReportRepository { .returningAll() .executeTakeFirst(); } + + deleteById(id: string) { + return this.db.deleteFrom('integrity_report').where('id', '=', id).execute(); + } + + deleteByIds(ids: string[]) { + return this.db.deleteFrom('integrity_report').where('id', 'in', ids).execute(); + } } diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index c4db5d49e7..5a093f0828 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -154,23 +154,8 @@ export class IntegrityService extends BaseService { this.logger.log(`Scanning for missing files...`); const assetPaths = this.assetJobRepository.streamAssetPaths(); - const assetFilePaths = this.assetJobRepository.streamAssetFilePaths(); - async function* paths() { - for await (const { originalPath, encodedVideoPath } of assetPaths) { - yield originalPath; - - if (encodedVideoPath) { - yield encodedVideoPath; - } - } - - for await (const { path } of assetFilePaths) { - yield path; - } - } - - async function* chunk(generator: AsyncGenerator, n: number) { + async function* chunk(generator: AsyncIterableIterator, n: number) { let chunk: T[] = []; for await (const item of generator) { chunk.push(item); @@ -187,7 +172,7 @@ export class IntegrityService extends BaseService { } let total = 0; - for await (const batchPaths of chunk(paths(), JOBS_LIBRARY_PAGINATION_SIZE)) { + for await (const batchPaths of chunk(assetPaths, JOBS_LIBRARY_PAGINATION_SIZE)) { await this.jobRepository.queue({ name: JobName.IntegrityMissingFiles, data: { @@ -206,22 +191,31 @@ export class IntegrityService extends BaseService { async handleMissingFiles({ paths }: IIntegrityMissingFilesJob): Promise { this.logger.log(`Processing batch of ${paths.length} files to check if they are missing.`); - const result = await Promise.all( - paths.map((path) => - stat(path) - .then(() => void 0) - .catch(() => path), + const results = await Promise.all( + paths.map((file) => + stat(file.path) + .then(() => ({ ...file, exists: true })) + .catch(() => ({ ...file, exists: false })), ), ); - const missingFiles = result.filter((path) => path) as string[]; + const outdatedReports = results + .filter(({ exists, reportId }) => exists && reportId) + .map(({ reportId }) => reportId!); - await this.integrityReportRepository.create( - missingFiles.map((path) => ({ - type: IntegrityReportType.MissingFile, - path, - })), - ); + if (outdatedReports.length) { + await this.integrityReportRepository.deleteByIds(outdatedReports); + } + + const missingFiles = results.filter(({ exists }) => !exists); + if (missingFiles.length) { + await this.integrityReportRepository.create( + missingFiles.map(({ path }) => ({ + type: IntegrityReportType.MissingFile, + path, + })), + ); + } this.logger.log(`Processed ${paths.length} and found ${missingFiles.length} missing file(s).`); return JobStatus.Success; @@ -264,7 +258,7 @@ export class IntegrityService extends BaseService { endMarker = startMarker; startMarker = undefined; - for await (const { originalPath, checksum, createdAt } of assets) { + for await (const { originalPath, checksum, createdAt, reportId } of assets) { try { const hash = createHash('sha1'); @@ -278,10 +272,22 @@ export class IntegrityService extends BaseService { }), ]); - if (!checksum.equals(hash.digest())) { + if (checksum.equals(hash.digest())) { + if (reportId) { + await this.integrityReportRepository.deleteById(reportId); + } + } else { throw new Error('File failed checksum'); } } catch (error) { + if ((error as { code?: string }).code === 'ENOENT') { + if (reportId) { + await this.integrityReportRepository.deleteById(reportId); + } + // missing file; handled by the missing files job + continue; + } + this.logger.warn('Failed to process a file: ' + error); await this.integrityReportRepository.create({ path: originalPath, diff --git a/server/src/types.ts b/server/src/types.ts index 183bdfc750..41c8feb4e9 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -288,7 +288,7 @@ export interface IIntegrityOrphanedFilesJob { } export interface IIntegrityMissingFilesJob { - paths: string[]; + paths: { path: string; reportId: string | null }[]; } export interface JobCounts { From ef7d8e94fa00eb26e9f5cc4992a616b024274981 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 15:40:14 +0000 Subject: [PATCH 08/50] feat: check orphaned file reports are not out of date --- open-api/immich-openapi-specs.json | 1 + server/src/enum.ts | 1 + .../src/repositories/asset-job.repository.ts | 5 ++ server/src/services/integrity.service.ts | 79 ++++++++++++++----- server/src/types.ts | 7 +- 5 files changed, 70 insertions(+), 23 deletions(-) diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index cc311e9335..9fe4664352 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -16663,6 +16663,7 @@ "WorkflowRun", "IntegrityOrphanedFilesQueueAll", "IntegrityOrphanedFiles", + "IntegrityOrphanedCheckReports", "IntegrityMissingFilesQueueAll", "IntegrityMissingFiles", "IntegrityChecksumFiles" diff --git a/server/src/enum.ts b/server/src/enum.ts index 145067b5e3..40f7f45495 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -648,6 +648,7 @@ export enum JobName { // Integrity IntegrityOrphanedFilesQueueAll = 'IntegrityOrphanedFilesQueueAll', IntegrityOrphanedFiles = 'IntegrityOrphanedFiles', + IntegrityOrphanedCheckReports = 'IntegrityOrphanedCheckReports', IntegrityMissingFilesQueueAll = 'IntegrityMissingFilesQueueAll', IntegrityMissingFiles = 'IntegrityMissingFiles', IntegrityChecksumFiles = 'IntegrityChecksumFiles', diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 79727520b1..8d5292ca0f 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -319,6 +319,11 @@ export class AssetJobRepository { .stream(); } + @GenerateSql({ params: [DummyValue.STRING], stream: true }) + streamIntegrityReports(type: IntegrityReportType) { + return this.db.selectFrom('integrity_report').select(['id as reportId', 'path']).where('type', '=', type).stream(); + } + @GenerateSql({ params: [], stream: true }) streamForVideoConversion(force?: boolean) { return this.db diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index 5a093f0828..924957f52a 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -18,7 +18,23 @@ import { } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; -import { IIntegrityMissingFilesJob, IIntegrityOrphanedFilesJob } from 'src/types'; +import { IIntegrityOrphanedFilesJob, IIntegrityPathWithReportJob } from 'src/types'; + +async function* chunk(generator: AsyncIterableIterator, n: number) { + let chunk: T[] = []; + for await (const item of generator) { + chunk.push(item); + + if (chunk.length === n) { + yield chunk; + chunk = []; + } + } + + if (chunk.length) { + yield chunk; + } +} @Injectable() export class IntegrityService extends BaseService { @@ -72,6 +88,23 @@ export class IntegrityService extends BaseService { @OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.BackgroundTask }) async handleOrphanedFilesQueueAll(): Promise { + this.logger.log(`Checking for out of date orphaned file reports...`); + + const reports = this.assetJobRepository.streamIntegrityReports(IntegrityReportType.OrphanFile); + + let total = 0; + for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityOrphanedCheckReports, + data: { + items: batchReports, + }, + }); + + total += batchReports.length; + this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`); + } + this.logger.log(`Scanning for orphaned files...`); const assetPaths = this.storageRepository.walk({ @@ -98,7 +131,7 @@ export class IntegrityService extends BaseService { } } - let total = 0; + total = 0; for await (const [batchType, batchPaths] of paths()) { await this.jobRepository.queue({ name: JobName.IntegrityOrphanedFiles, @@ -149,34 +182,40 @@ export class IntegrityService extends BaseService { return JobStatus.Success; } + @OnJob({ name: JobName.IntegrityOrphanedCheckReports, queue: QueueName.BackgroundTask }) + async handleOrphanedCheckReports({ items: paths }: IIntegrityPathWithReportJob): Promise { + this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`); + + const results = await Promise.all( + paths.map(({ reportId, path }) => + stat(path) + .then(() => reportId) + .catch(() => void 0), + ), + ); + + const reportIds = results.filter((reportId) => reportId) as string[]; + + if (reportIds.length) { + await this.integrityReportRepository.deleteByIds(reportIds); + } + + this.logger.log(`Processed ${paths.length} and found ${reportIds.length} orphaned file(s).`); + return JobStatus.Success; + } + @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.BackgroundTask }) async handleMissingFilesQueueAll(): Promise { this.logger.log(`Scanning for missing files...`); const assetPaths = this.assetJobRepository.streamAssetPaths(); - async function* chunk(generator: AsyncIterableIterator, n: number) { - let chunk: T[] = []; - for await (const item of generator) { - chunk.push(item); - - if (chunk.length === n) { - yield chunk; - chunk = []; - } - } - - if (chunk.length) { - yield chunk; - } - } - let total = 0; for await (const batchPaths of chunk(assetPaths, JOBS_LIBRARY_PAGINATION_SIZE)) { await this.jobRepository.queue({ name: JobName.IntegrityMissingFiles, data: { - paths: batchPaths, + items: batchPaths, }, }); @@ -188,7 +227,7 @@ export class IntegrityService extends BaseService { } @OnJob({ name: JobName.IntegrityMissingFiles, queue: QueueName.BackgroundTask }) - async handleMissingFiles({ paths }: IIntegrityMissingFilesJob): Promise { + async handleMissingFiles({ items: paths }: IIntegrityPathWithReportJob): Promise { this.logger.log(`Processing batch of ${paths.length} files to check if they are missing.`); const results = await Promise.all( diff --git a/server/src/types.ts b/server/src/types.ts index 41c8feb4e9..b02626ce26 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -287,8 +287,8 @@ export interface IIntegrityOrphanedFilesJob { paths: string[]; } -export interface IIntegrityMissingFilesJob { - paths: { path: string; reportId: string | null }[]; +export interface IIntegrityPathWithReportJob { + items: { path: string; reportId: string | null }[]; } export interface JobCounts { @@ -405,8 +405,9 @@ export type JobItem = // Integrity | { name: JobName.IntegrityOrphanedFilesQueueAll; data: IBaseJob } | { name: JobName.IntegrityOrphanedFiles; data: IIntegrityOrphanedFilesJob } + | { name: JobName.IntegrityOrphanedCheckReports; data: IIntegrityPathWithReportJob } | { name: JobName.IntegrityMissingFilesQueueAll; data: IBaseJob } - | { name: JobName.IntegrityMissingFiles; data: IIntegrityMissingFilesJob } + | { name: JobName.IntegrityMissingFiles; data: IIntegrityPathWithReportJob } | { name: JobName.IntegrityChecksumFiles; data: IBaseJob }; export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number]; From 1744237aeb3f9428527dd2218ad634b7443b914e Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 15:40:44 +0000 Subject: [PATCH 09/50] chore: open api --- mobile/openapi/lib/model/job_name.dart | 3 +++ open-api/typescript-sdk/src/fetch-client.ts | 1 + 2 files changed, 4 insertions(+) diff --git a/mobile/openapi/lib/model/job_name.dart b/mobile/openapi/lib/model/job_name.dart index bd2e599529..c449ebb69e 100644 --- a/mobile/openapi/lib/model/job_name.dart +++ b/mobile/openapi/lib/model/job_name.dart @@ -80,6 +80,7 @@ class JobName { static const workflowRun = JobName._(r'WorkflowRun'); static const integrityOrphanedFilesQueueAll = JobName._(r'IntegrityOrphanedFilesQueueAll'); static const integrityOrphanedFiles = JobName._(r'IntegrityOrphanedFiles'); + static const integrityOrphanedCheckReports = JobName._(r'IntegrityOrphanedCheckReports'); static const integrityMissingFilesQueueAll = JobName._(r'IntegrityMissingFilesQueueAll'); static const integrityMissingFiles = JobName._(r'IntegrityMissingFiles'); static const integrityChecksumFiles = JobName._(r'IntegrityChecksumFiles'); @@ -143,6 +144,7 @@ class JobName { workflowRun, integrityOrphanedFilesQueueAll, integrityOrphanedFiles, + integrityOrphanedCheckReports, integrityMissingFilesQueueAll, integrityMissingFiles, integrityChecksumFiles, @@ -241,6 +243,7 @@ class JobNameTypeTransformer { case r'WorkflowRun': return JobName.workflowRun; case r'IntegrityOrphanedFilesQueueAll': return JobName.integrityOrphanedFilesQueueAll; case r'IntegrityOrphanedFiles': return JobName.integrityOrphanedFiles; + case r'IntegrityOrphanedCheckReports': return JobName.integrityOrphanedCheckReports; case r'IntegrityMissingFilesQueueAll': return JobName.integrityMissingFilesQueueAll; case r'IntegrityMissingFiles': return JobName.integrityMissingFiles; case r'IntegrityChecksumFiles': return JobName.integrityChecksumFiles; diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index b3242b5d29..3fe3d3a51c 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -5489,6 +5489,7 @@ export enum JobName { WorkflowRun = "WorkflowRun", IntegrityOrphanedFilesQueueAll = "IntegrityOrphanedFilesQueueAll", IntegrityOrphanedFiles = "IntegrityOrphanedFiles", + IntegrityOrphanedCheckReports = "IntegrityOrphanedCheckReports", IntegrityMissingFilesQueueAll = "IntegrityMissingFilesQueueAll", IntegrityMissingFiles = "IntegrityMissingFiles", IntegrityChecksumFiles = "IntegrityChecksumFiles" From 93860238af3bb1fbe857465d84507385ab4a9495 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 16:05:26 +0000 Subject: [PATCH 10/50] feat: add config options & cron entries for checks --- mobile/openapi/README.md | 3 + mobile/openapi/lib/api.dart | 3 + mobile/openapi/lib/api_client.dart | 6 + .../openapi/lib/model/system_config_dto.dart | 10 +- .../model/system_config_integrity_checks.dart | 115 ++++++++++++++++ .../system_config_integrity_checksum_job.dart | 123 ++++++++++++++++++ .../model/system_config_integrity_job.dart | 107 +++++++++++++++ open-api/immich-openapi-specs.json | 61 +++++++++ open-api/typescript-sdk/src/fetch-client.ts | 16 +++ server/src/config.ts | 32 +++++ server/src/dtos/system-config.dto.ts | 42 ++++++ server/src/enum.ts | 1 + server/src/services/integrity.service.ts | 87 +++++++++---- 13 files changed, 583 insertions(+), 23 deletions(-) create mode 100644 mobile/openapi/lib/model/system_config_integrity_checks.dart create mode 100644 mobile/openapi/lib/model/system_config_integrity_checksum_job.dart create mode 100644 mobile/openapi/lib/model/system_config_integrity_job.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 268c4849c5..391c7c3759 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -569,6 +569,9 @@ Class | Method | HTTP request | Description - [SystemConfigGeneratedFullsizeImageDto](doc//SystemConfigGeneratedFullsizeImageDto.md) - [SystemConfigGeneratedImageDto](doc//SystemConfigGeneratedImageDto.md) - [SystemConfigImageDto](doc//SystemConfigImageDto.md) + - [SystemConfigIntegrityChecks](doc//SystemConfigIntegrityChecks.md) + - [SystemConfigIntegrityChecksumJob](doc//SystemConfigIntegrityChecksumJob.md) + - [SystemConfigIntegrityJob](doc//SystemConfigIntegrityJob.md) - [SystemConfigJobDto](doc//SystemConfigJobDto.md) - [SystemConfigLibraryDto](doc//SystemConfigLibraryDto.md) - [SystemConfigLibraryScanDto](doc//SystemConfigLibraryScanDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 21730074aa..641000daf7 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -321,6 +321,9 @@ part 'model/system_config_faces_dto.dart'; part 'model/system_config_generated_fullsize_image_dto.dart'; part 'model/system_config_generated_image_dto.dart'; part 'model/system_config_image_dto.dart'; +part 'model/system_config_integrity_checks.dart'; +part 'model/system_config_integrity_checksum_job.dart'; +part 'model/system_config_integrity_job.dart'; part 'model/system_config_job_dto.dart'; part 'model/system_config_library_dto.dart'; part 'model/system_config_library_scan_dto.dart'; diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 041be67015..ae2fa85a93 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -690,6 +690,12 @@ class ApiClient { return SystemConfigGeneratedImageDto.fromJson(value); case 'SystemConfigImageDto': return SystemConfigImageDto.fromJson(value); + case 'SystemConfigIntegrityChecks': + return SystemConfigIntegrityChecks.fromJson(value); + case 'SystemConfigIntegrityChecksumJob': + return SystemConfigIntegrityChecksumJob.fromJson(value); + case 'SystemConfigIntegrityJob': + return SystemConfigIntegrityJob.fromJson(value); case 'SystemConfigJobDto': return SystemConfigJobDto.fromJson(value); case 'SystemConfigLibraryDto': diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dart index 38dbb30f0c..3a5e15b030 100644 --- a/mobile/openapi/lib/model/system_config_dto.dart +++ b/mobile/openapi/lib/model/system_config_dto.dart @@ -16,6 +16,7 @@ class SystemConfigDto { required this.backup, required this.ffmpeg, required this.image, + required this.integrityChecks, required this.job, required this.library_, required this.logging, @@ -42,6 +43,8 @@ class SystemConfigDto { SystemConfigImageDto image; + SystemConfigIntegrityChecks integrityChecks; + SystemConfigJobDto job; SystemConfigLibraryDto library_; @@ -83,6 +86,7 @@ class SystemConfigDto { other.backup == backup && other.ffmpeg == ffmpeg && other.image == image && + other.integrityChecks == integrityChecks && other.job == job && other.library_ == library_ && other.logging == logging && @@ -108,6 +112,7 @@ class SystemConfigDto { (backup.hashCode) + (ffmpeg.hashCode) + (image.hashCode) + + (integrityChecks.hashCode) + (job.hashCode) + (library_.hashCode) + (logging.hashCode) + @@ -128,13 +133,14 @@ class SystemConfigDto { (user.hashCode); @override - String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]'; + String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, integrityChecks=$integrityChecks, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]'; Map toJson() { final json = {}; json[r'backup'] = this.backup; json[r'ffmpeg'] = this.ffmpeg; json[r'image'] = this.image; + json[r'integrityChecks'] = this.integrityChecks; json[r'job'] = this.job; json[r'library'] = this.library_; json[r'logging'] = this.logging; @@ -168,6 +174,7 @@ class SystemConfigDto { backup: SystemConfigBackupsDto.fromJson(json[r'backup'])!, ffmpeg: SystemConfigFFmpegDto.fromJson(json[r'ffmpeg'])!, image: SystemConfigImageDto.fromJson(json[r'image'])!, + integrityChecks: SystemConfigIntegrityChecks.fromJson(json[r'integrityChecks'])!, job: SystemConfigJobDto.fromJson(json[r'job'])!, library_: SystemConfigLibraryDto.fromJson(json[r'library'])!, logging: SystemConfigLoggingDto.fromJson(json[r'logging'])!, @@ -236,6 +243,7 @@ class SystemConfigDto { 'backup', 'ffmpeg', 'image', + 'integrityChecks', 'job', 'library', 'logging', diff --git a/mobile/openapi/lib/model/system_config_integrity_checks.dart b/mobile/openapi/lib/model/system_config_integrity_checks.dart new file mode 100644 index 0000000000..37a75ed3d7 --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_checks.dart @@ -0,0 +1,115 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityChecks { + /// Returns a new [SystemConfigIntegrityChecks] instance. + SystemConfigIntegrityChecks({ + required this.checksumFiles, + required this.missingFiles, + required this.orphanedFiles, + }); + + SystemConfigIntegrityChecksumJob checksumFiles; + + SystemConfigIntegrityJob missingFiles; + + SystemConfigIntegrityJob orphanedFiles; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityChecks && + other.checksumFiles == checksumFiles && + other.missingFiles == missingFiles && + other.orphanedFiles == orphanedFiles; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (checksumFiles.hashCode) + + (missingFiles.hashCode) + + (orphanedFiles.hashCode); + + @override + String toString() => 'SystemConfigIntegrityChecks[checksumFiles=$checksumFiles, missingFiles=$missingFiles, orphanedFiles=$orphanedFiles]'; + + Map toJson() { + final json = {}; + json[r'checksumFiles'] = this.checksumFiles; + json[r'missingFiles'] = this.missingFiles; + json[r'orphanedFiles'] = this.orphanedFiles; + return json; + } + + /// Returns a new [SystemConfigIntegrityChecks] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityChecks? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityChecks"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityChecks( + checksumFiles: SystemConfigIntegrityChecksumJob.fromJson(json[r'checksumFiles'])!, + missingFiles: SystemConfigIntegrityJob.fromJson(json[r'missingFiles'])!, + orphanedFiles: SystemConfigIntegrityJob.fromJson(json[r'orphanedFiles'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityChecks.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityChecks.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityChecks-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityChecks.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'checksumFiles', + 'missingFiles', + 'orphanedFiles', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart b/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart new file mode 100644 index 0000000000..16b64bf2b9 --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart @@ -0,0 +1,123 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityChecksumJob { + /// Returns a new [SystemConfigIntegrityChecksumJob] instance. + SystemConfigIntegrityChecksumJob({ + required this.cronExpression, + required this.enabled, + required this.percentageLimit, + required this.timeLimit, + }); + + String cronExpression; + + bool enabled; + + num percentageLimit; + + num timeLimit; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityChecksumJob && + other.cronExpression == cronExpression && + other.enabled == enabled && + other.percentageLimit == percentageLimit && + other.timeLimit == timeLimit; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (cronExpression.hashCode) + + (enabled.hashCode) + + (percentageLimit.hashCode) + + (timeLimit.hashCode); + + @override + String toString() => 'SystemConfigIntegrityChecksumJob[cronExpression=$cronExpression, enabled=$enabled, percentageLimit=$percentageLimit, timeLimit=$timeLimit]'; + + Map toJson() { + final json = {}; + json[r'cronExpression'] = this.cronExpression; + json[r'enabled'] = this.enabled; + json[r'percentageLimit'] = this.percentageLimit; + json[r'timeLimit'] = this.timeLimit; + return json; + } + + /// Returns a new [SystemConfigIntegrityChecksumJob] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityChecksumJob? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityChecksumJob"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityChecksumJob( + cronExpression: mapValueOfType(json, r'cronExpression')!, + enabled: mapValueOfType(json, r'enabled')!, + percentageLimit: num.parse('${json[r'percentageLimit']}'), + timeLimit: num.parse('${json[r'timeLimit']}'), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityChecksumJob.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityChecksumJob.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityChecksumJob-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityChecksumJob.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'cronExpression', + 'enabled', + 'percentageLimit', + 'timeLimit', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_integrity_job.dart b/mobile/openapi/lib/model/system_config_integrity_job.dart new file mode 100644 index 0000000000..08eb559c7f --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_job.dart @@ -0,0 +1,107 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityJob { + /// Returns a new [SystemConfigIntegrityJob] instance. + SystemConfigIntegrityJob({ + required this.cronExpression, + required this.enabled, + }); + + String cronExpression; + + bool enabled; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityJob && + other.cronExpression == cronExpression && + other.enabled == enabled; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (cronExpression.hashCode) + + (enabled.hashCode); + + @override + String toString() => 'SystemConfigIntegrityJob[cronExpression=$cronExpression, enabled=$enabled]'; + + Map toJson() { + final json = {}; + json[r'cronExpression'] = this.cronExpression; + json[r'enabled'] = this.enabled; + return json; + } + + /// Returns a new [SystemConfigIntegrityJob] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityJob? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityJob"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityJob( + cronExpression: mapValueOfType(json, r'cronExpression')!, + enabled: mapValueOfType(json, r'enabled')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityJob.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityJob.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityJob-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityJob.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'cronExpression', + 'enabled', + }; +} + diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 9fe4664352..f1a61bc4ea 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -21237,6 +21237,9 @@ "image": { "$ref": "#/components/schemas/SystemConfigImageDto" }, + "integrityChecks": { + "$ref": "#/components/schemas/SystemConfigIntegrityChecks" + }, "job": { "$ref": "#/components/schemas/SystemConfigJobDto" }, @@ -21296,6 +21299,7 @@ "backup", "ffmpeg", "image", + "integrityChecks", "job", "library", "logging", @@ -21542,6 +21546,63 @@ ], "type": "object" }, + "SystemConfigIntegrityChecks": { + "properties": { + "checksumFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityChecksumJob" + }, + "missingFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityJob" + }, + "orphanedFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityJob" + } + }, + "required": [ + "checksumFiles", + "missingFiles", + "orphanedFiles" + ], + "type": "object" + }, + "SystemConfigIntegrityChecksumJob": { + "properties": { + "cronExpression": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "percentageLimit": { + "type": "number" + }, + "timeLimit": { + "type": "number" + } + }, + "required": [ + "cronExpression", + "enabled", + "percentageLimit", + "timeLimit" + ], + "type": "object" + }, + "SystemConfigIntegrityJob": { + "properties": { + "cronExpression": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "cronExpression", + "enabled" + ], + "type": "object" + }, "SystemConfigJobDto": { "properties": { "backgroundTask": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 3fe3d3a51c..b33276a90b 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1454,6 +1454,21 @@ export type SystemConfigImageDto = { preview: SystemConfigGeneratedImageDto; thumbnail: SystemConfigGeneratedImageDto; }; +export type SystemConfigIntegrityChecksumJob = { + cronExpression: string; + enabled: boolean; + percentageLimit: number; + timeLimit: number; +}; +export type SystemConfigIntegrityJob = { + cronExpression: string; + enabled: boolean; +}; +export type SystemConfigIntegrityChecks = { + checksumFiles: SystemConfigIntegrityChecksumJob; + missingFiles: SystemConfigIntegrityJob; + orphanedFiles: SystemConfigIntegrityJob; +}; export type JobSettingsDto = { concurrency: number; }; @@ -1606,6 +1621,7 @@ export type SystemConfigDto = { backup: SystemConfigBackupsDto; ffmpeg: SystemConfigFFmpegDto; image: SystemConfigImageDto; + integrityChecks: SystemConfigIntegrityChecks; job: SystemConfigJobDto; library: SystemConfigLibraryDto; logging: SystemConfigLoggingDto; diff --git a/server/src/config.ts b/server/src/config.ts index c18acd79f8..bd3c745daf 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -46,6 +46,22 @@ export interface SystemConfig { accelDecode: boolean; tonemap: ToneMapping; }; + integrityChecks: { + missingFiles: { + enabled: boolean; + cronExpression: string; + }; + orphanedFiles: { + enabled: boolean; + cronExpression: string; + }; + checksumFiles: { + enabled: boolean; + cronExpression: string; + timeLimit: number; + percentageLimit: number; + }; + }; job: Record; logging: { enabled: boolean; @@ -222,6 +238,22 @@ export const defaults = Object.freeze({ accel: TranscodeHardwareAcceleration.Disabled, accelDecode: false, }, + integrityChecks: { + missingFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + }, + orphanedFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + }, + checksumFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + timeLimit: 60 * 60 * 1000, // 1 hour + percentageLimit: 1.0, // 100% of assets + }, + }, job: { [QueueName.BackgroundTask]: { concurrency: 5 }, [QueueName.SmartSearch]: { concurrency: 2 }, diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index c835073c31..5701f7eeba 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -38,6 +38,7 @@ const isOAuthEnabled = (config: SystemConfigOAuthDto) => config.enabled; const isOAuthOverrideEnabled = (config: SystemConfigOAuthDto) => config.mobileOverrideEnabled; const isEmailNotificationEnabled = (config: SystemConfigSmtpDto) => config.enabled; const isDatabaseBackupEnabled = (config: DatabaseBackupConfig) => config.enabled; +const isEnabledProperty = (config: { enabled: boolean }) => config.enabled; export class DatabaseBackupConfig { @ValidateBoolean() @@ -145,6 +146,42 @@ export class SystemConfigFFmpegDto { tonemap!: ToneMapping; } +class SystemConfigIntegrityJob { + @ValidateBoolean() + enabled!: boolean; + + @ValidateIf(isEnabledProperty) + @IsNotEmpty() + @IsCronExpression() + @IsString() + cronExpression!: string; +} + +class SystemConfigIntegrityChecksumJob extends SystemConfigIntegrityJob { + @IsInt() + timeLimit!: number; + + @IsNumber() + percentageLimit!: number; +} + +class SystemConfigIntegrityChecks { + @Type(() => SystemConfigIntegrityJob) + @ValidateNested() + @IsObject() + missingFiles!: SystemConfigIntegrityJob; + + @Type(() => SystemConfigIntegrityJob) + @ValidateNested() + @IsObject() + orphanedFiles!: SystemConfigIntegrityJob; + + @Type(() => SystemConfigIntegrityChecksumJob) + @ValidateNested() + @IsObject() + checksumFiles!: SystemConfigIntegrityChecksumJob; +} + class JobSettingsDto { @IsInt() @IsPositive() @@ -649,6 +686,11 @@ export class SystemConfigDto implements SystemConfig { @IsObject() ffmpeg!: SystemConfigFFmpegDto; + @Type(() => SystemConfigIntegrityChecks) + @ValidateNested() + @IsObject() + integrityChecks!: SystemConfigIntegrityChecks; + @Type(() => SystemConfigLoggingDto) @ValidateNested() @IsObject() diff --git a/server/src/enum.ts b/server/src/enum.ts index 40f7f45495..0ee2765741 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -694,6 +694,7 @@ export enum DatabaseLock { GetSystemConfig = 69, BackupDatabase = 42, MemoryCreation = 777, + IntegrityCheck = 67, } export enum MaintenanceAction { diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index 924957f52a..b876004385 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -8,6 +8,7 @@ import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent, OnJob } from 'src/decorators'; import { + DatabaseLock, ImmichWorker, IntegrityReportType, JobName, @@ -19,6 +20,7 @@ import { import { ArgOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; import { IIntegrityOrphanedFilesJob, IIntegrityPathWithReportJob } from 'src/types'; +import { handlePromiseError } from 'src/utils/misc'; async function* chunk(generator: AsyncIterableIterator, n: number) { let chunk: T[] = []; @@ -38,25 +40,49 @@ async function* chunk(generator: AsyncIterableIterator, n: number) { @Injectable() export class IntegrityService extends BaseService { - // private backupLock = false; + private integrityLock = false; @OnEvent({ name: 'ConfigInit', workers: [ImmichWorker.Microservices] }) async onConfigInit({ newConfig: { - backup: { database }, + integrityChecks: { orphanedFiles, missingFiles, checksumFiles }, }, }: ArgOf<'ConfigInit'>) { - // this.backupLock = await this.databaseRepository.tryLock(DatabaseLock.BackupDatabase); - // if (this.backupLock) { - // this.cronRepository.create({ - // name: 'backupDatabase', - // expression: database.cronExpression, - // onTick: () => handlePromiseError(this.jobRepository.queue({ name: JobName.DatabaseBackup }), this.logger), - // start: database.enabled, - // }); - // } + this.integrityLock = await this.databaseRepository.tryLock(DatabaseLock.IntegrityCheck); + if (this.integrityLock) { + this.cronRepository.create({ + name: 'integrityOrphanedFiles', + expression: orphanedFiles.cronExpression, + onTick: () => + handlePromiseError( + this.jobRepository.queue({ name: JobName.IntegrityOrphanedFilesQueueAll, data: {} }), + this.logger, + ), + start: orphanedFiles.enabled, + }); - setTimeout(() => { + this.cronRepository.create({ + name: 'integrityMissingFiles', + expression: missingFiles.cronExpression, + onTick: () => + handlePromiseError( + this.jobRepository.queue({ name: JobName.IntegrityMissingFilesQueueAll, data: {} }), + this.logger, + ), + start: missingFiles.enabled, + }); + + this.cronRepository.create({ + name: 'integrityChecksumFiles', + expression: checksumFiles.cronExpression, + onTick: () => + handlePromiseError(this.jobRepository.queue({ name: JobName.IntegrityChecksumFiles, data: {} }), this.logger), + start: checksumFiles.enabled, + }); + } + + // debug: run on boot + setImmediate(() => { this.jobRepository.queue({ name: JobName.IntegrityOrphanedFilesQueueAll, data: {}, @@ -71,19 +97,36 @@ export class IntegrityService extends BaseService { name: JobName.IntegrityChecksumFiles, data: {}, }); - }, 1000); + }); } @OnEvent({ name: 'ConfigUpdate', server: true }) - async onConfigUpdate({ newConfig: { backup } }: ArgOf<'ConfigUpdate'>) { - // if (!this.backupLock) { - // return; - // } - // this.cronRepository.update({ - // name: 'backupDatabase', - // expression: backup.database.cronExpression, - // start: backup.database.enabled, - // }); + async onConfigUpdate({ + newConfig: { + integrityChecks: { orphanedFiles, missingFiles, checksumFiles }, + }, + }: ArgOf<'ConfigUpdate'>) { + if (!this.integrityLock) { + return; + } + + this.cronRepository.update({ + name: 'integrityOrphanedFiles', + expression: orphanedFiles.cronExpression, + start: orphanedFiles.enabled, + }); + + this.cronRepository.update({ + name: 'integrityMissingFiles', + expression: missingFiles.cronExpression, + start: missingFiles.enabled, + }); + + this.cronRepository.update({ + name: 'integrityChecksumFiles', + expression: checksumFiles.cronExpression, + start: checksumFiles.enabled, + }); } @OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.BackgroundTask }) From 251631948b3b45c9cc572a164c26b4de9cd1942b Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 16:11:19 +0000 Subject: [PATCH 11/50] fix: mock the new repository --- server/test/utils.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/test/utils.ts b/server/test/utils.ts index 77853f897a..850699b51c 100644 --- a/server/test/utils.ts +++ b/server/test/utils.ts @@ -31,6 +31,7 @@ import { DownloadRepository } from 'src/repositories/download.repository'; import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityReportRepository } from 'src/repositories/integrity-report.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -225,6 +226,7 @@ export type ServiceOverrides = { duplicateRepository: DuplicateRepository; email: EmailRepository; event: EventRepository; + integrityReport: IntegrityReportRepository; job: JobRepository; library: LibraryRepository; logger: LoggingRepository; @@ -298,6 +300,7 @@ export const getMocks = () => { email: automock(EmailRepository, { args: [loggerMock] }), // eslint-disable-next-line no-sparse-arrays event: automock(EventRepository, { args: [, , loggerMock], strict: false }), + integrityReport: automock(IntegrityReportRepository, { strict: false }), job: newJobRepositoryMock(), apiKey: automock(ApiKeyRepository), library: automock(LibraryRepository, { strict: false }), @@ -366,6 +369,7 @@ export const newTestService = ( overrides.duplicateRepository || (mocks.duplicateRepository as As), overrides.email || (mocks.email as As), overrides.event || (mocks.event as As), + overrides.integrityReport || (mocks.integrityReport as As), overrides.job || (mocks.job as As), overrides.library || (mocks.library as As), overrides.machineLearning || (mocks.machineLearning as As), From 919eb839efbd96020635e4d594fe4d03f6791ab9 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 16:14:03 +0000 Subject: [PATCH 12/50] revert: override migration db url --- server/src/bin/migrations.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/src/bin/migrations.ts b/server/src/bin/migrations.ts index 8c873c4274..588f358023 100644 --- a/server/src/bin/migrations.ts +++ b/server/src/bin/migrations.ts @@ -62,10 +62,6 @@ const main = async () => { const getDatabaseClient = () => { const configRepository = new ConfigRepository(); const { database } = configRepository.getEnv(); - database.config = { - connectionType: 'url', - url: 'postgres://postgres:postgres@database:5432/immich', - }; return new Kysely(getKyselyConfig(database.config)); }; @@ -134,10 +130,6 @@ const create = (path: string, up: string[], down: string[]) => { const compare = async () => { const configRepository = new ConfigRepository(); const { database } = configRepository.getEnv(); - database.config = { - connectionType: 'url', - url: 'postgres://postgres:postgres@database:5432/immich', - }; const db = postgres(asPostgresConnectionConfig(database.config)); const source = schemaFromCode({ overrides: true, namingStrategy: 'default' }); From 446268373978ecb8f55695873b6936aeaaa25dd1 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 16:19:34 +0000 Subject: [PATCH 13/50] chore: generate SQL queries --- server/src/queries/asset.job.repository.sql | 78 +++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql index ebfd1a08c9..b901327604 100644 --- a/server/src/queries/asset.job.repository.sql +++ b/server/src/queries/asset.job.repository.sql @@ -386,6 +386,84 @@ from where "asset"."id" = $2 +-- AssetJobRepository.getAssetPathsByPaths +select + "originalPath", + "encodedVideoPath" +from + "asset" +where + ( + "originalPath" in $1 + or "encodedVideoPath" in $2 + ) + +-- AssetJobRepository.getAssetFilePathsByPaths +select + "path" +from + "asset_file" +where + "path" in $1 + +-- AssetJobRepository.getAssetCount +select + count(*) as "count" +from + "asset" + +-- AssetJobRepository.streamAssetPaths +select + "allPaths"."path" as "path", + "integrity_report"."path" as "reportId" +from + ( + select + "originalPath" as "path" + from + "asset" + union all + select + "encodedVideoPath" as "path" + from + "asset" + where + "encodedVideoPath" is not null + and "encodedVideoPath" != $1 + union all + select + "path" + from + "asset_file" + ) as "allPaths" + left join "integrity_report" on "integrity_report"."path" = "allPaths"."path" + and "integrity_report"."type" = $2 + +-- AssetJobRepository.streamAssetChecksums +select + "asset"."originalPath", + "asset"."checksum", + "asset"."createdAt", + "integrity_report"."id" as "reportId" +from + "asset" + left join "integrity_report" on "integrity_report"."path" = "asset"."originalPath" + and "integrity_report"."type" = $1 +where + "createdAt" >= $2 + and "createdAt" <= $3 +order by + "createdAt" asc + +-- AssetJobRepository.streamIntegrityReports +select + "id" as "reportId", + "path" +from + "integrity_report" +where + "type" = $1 + -- AssetJobRepository.streamForVideoConversion select "asset"."id" From 03276de6b2f30bbc907bcabd6f01b7909313cab0 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 16:34:28 +0000 Subject: [PATCH 14/50] fix: add integrity report repository to service depends. --- server/src/services/base.service.ts | 1 + .../src/services/system-config.service.spec.ts | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index e311a860e3..67522ec38b 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -80,6 +80,7 @@ export const BASE_SERVICE_DEPENDENCIES = [ DuplicateRepository, EmailRepository, EventRepository, + IntegrityReportRepository, JobRepository, LibraryRepository, MachineLearningRepository, diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index fbdd655bbc..8f5ed468de 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -72,6 +72,22 @@ const updatedConfig = Object.freeze({ accelDecode: false, tonemap: ToneMapping.Hable, }, + integrityChecks: { + orphanedFiles: { + enabled: true, + cronExpression: '0 03 * * *', + }, + missingFiles: { + enabled: true, + cronExpression: '0 03 * * *', + }, + checksumFiles: { + enabled: true, + cronExpression: '0 03 * * *', + timeLimit: 60 * 60 * 1000, + percentageLimit: 1.0, + }, + }, logging: { enabled: true, level: LogLevel.Log, From 8db6132669b313b180a1f6fea0a0eef51b39c4c7 Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 16:42:46 +0000 Subject: [PATCH 15/50] fix: add mock for asset repo. --- server/test/repositories/asset.repository.mock.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index e735b37564..49815358ab 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -20,6 +20,8 @@ export const newAssetRepositoryMock = (): Mocked Date: Thu, 27 Nov 2025 17:23:54 +0000 Subject: [PATCH 16/50] feat: draft controller entry chore: lint & format --- open-api/immich-openapi-specs.json | 79 +++++++++++++++++++ server/src/config.ts | 2 +- .../src/controllers/maintenance.controller.ts | 21 ++++- server/src/dtos/maintenance.dto.ts | 22 +++++- server/src/repositories/asset.repository.ts | 4 +- .../integrity-report.repository.ts | 11 +++ server/src/services/integrity.service.ts | 22 +++--- server/src/services/maintenance.service.ts | 10 ++- .../services/system-config.service.spec.ts | 2 +- 9 files changed, 154 insertions(+), 19 deletions(-) diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index f1a61bc4ea..0d87727e8a 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -323,6 +323,51 @@ } }, "/admin/maintenance": { + "get": { + "description": "...", + "operationId": "getIntegrityReport", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MaintenanceIntegrityReportResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get integrity report", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v9.9.9", + "state": "Added" + }, + { + "version": "v9.9.9", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + }, "post": { "description": "Put Immich into or take it out of maintenance mode", "operationId": "setMaintenanceMode", @@ -16920,6 +16965,40 @@ ], "type": "object" }, + "MaintenanceIntegrityReportDto": { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "orphan_file", + "missing_file", + "checksum_mismatch" + ], + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "type": "object" + }, + "MaintenanceIntegrityReportResponseDto": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/MaintenanceIntegrityReportDto" + }, + "type": "array" + } + }, + "required": [ + "items" + ], + "type": "object" + }, "MaintenanceLoginDto": { "properties": { "token": { diff --git a/server/src/config.ts b/server/src/config.ts index bd3c745daf..bca84ef40b 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -251,7 +251,7 @@ export const defaults = Object.freeze({ enabled: true, cronExpression: CronExpression.EVERY_DAY_AT_3AM, timeLimit: 60 * 60 * 1000, // 1 hour - percentageLimit: 1.0, // 100% of assets + percentageLimit: 1, // 100% of assets }, }, job: { diff --git a/server/src/controllers/maintenance.controller.ts b/server/src/controllers/maintenance.controller.ts index 7b2aa17582..36e8003521 100644 --- a/server/src/controllers/maintenance.controller.ts +++ b/server/src/controllers/maintenance.controller.ts @@ -1,9 +1,15 @@ -import { BadRequestException, Body, Controller, Post, Res } from '@nestjs/common'; +import { BadRequestException, Body, Controller, Get, Post, Res } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { Response } from 'express'; import { Endpoint, HistoryBuilder } from 'src/decorators'; import { AuthDto } from 'src/dtos/auth.dto'; -import { MaintenanceAuthDto, MaintenanceLoginDto, SetMaintenanceModeDto } from 'src/dtos/maintenance.dto'; +import { + MaintenanceAuthDto, + MaintenanceGetIntegrityReportDto, + MaintenanceIntegrityReportResponseDto, + MaintenanceLoginDto, + SetMaintenanceModeDto, +} from 'src/dtos/maintenance.dto'; import { ApiTag, ImmichCookie, MaintenanceAction, Permission } from 'src/enum'; import { Auth, Authenticated, GetLoginDetails } from 'src/middleware/auth.guard'; import { LoginDetails } from 'src/services/auth.service'; @@ -46,4 +52,15 @@ export class MaintenanceController { }); } } + + @Get() + @Endpoint({ + summary: 'Get integrity report', + description: '...', + history: new HistoryBuilder().added('v9.9.9').alpha('v9.9.9'), + }) + @Authenticated({ permission: Permission.Maintenance, admin: true }) + getIntegrityReport(dto: MaintenanceGetIntegrityReportDto): Promise { + return this.service.getIntegrityReport(dto); + } } diff --git a/server/src/dtos/maintenance.dto.ts b/server/src/dtos/maintenance.dto.ts index fe6960c0a4..6d7f15f55c 100644 --- a/server/src/dtos/maintenance.dto.ts +++ b/server/src/dtos/maintenance.dto.ts @@ -1,4 +1,5 @@ -import { MaintenanceAction } from 'src/enum'; +import { IsEnum } from 'class-validator'; +import { IntegrityReportType, MaintenanceAction } from 'src/enum'; import { ValidateEnum, ValidateString } from 'src/validation'; export class SetMaintenanceModeDto { @@ -14,3 +15,22 @@ export class MaintenanceLoginDto { export class MaintenanceAuthDto { username!: string; } + +export class MaintenanceGetIntegrityReportDto { + // todo: paginate + // @IsInt() + // @Min(1) + // @Type(() => Number) + // @Optional() + // page?: number; +} + +class MaintenanceIntegrityReportDto { + @IsEnum(IntegrityReportType) + type!: IntegrityReportType; + path!: string; +} + +export class MaintenanceIntegrityReportResponseDto { + items!: MaintenanceIntegrityReportDto[]; +} diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 226d021745..afdc29876e 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -382,11 +382,11 @@ export class AssetRepository { return items.map((asset) => asset.deviceAssetId); } - async getAllAssetPaths() { + getAllAssetPaths() { return this.db.selectFrom('asset').select(['originalPath', 'encodedVideoPath']).stream(); } - async getAllAssetFilePaths() { + getAllAssetFilePaths() { return this.db.selectFrom('asset_file').select(['path']).stream(); } diff --git a/server/src/repositories/integrity-report.repository.ts b/server/src/repositories/integrity-report.repository.ts index 25194731d6..af36f051de 100644 --- a/server/src/repositories/integrity-report.repository.ts +++ b/server/src/repositories/integrity-report.repository.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { Insertable, Kysely } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; +import { MaintenanceGetIntegrityReportDto, MaintenanceIntegrityReportResponseDto } from 'src/dtos/maintenance.dto'; import { DB } from 'src/schema'; import { IntegrityReportTable } from 'src/schema/tables/integrity-report.table'; @@ -17,6 +18,16 @@ export class IntegrityReportRepository { .executeTakeFirst(); } + async getIntegrityReport(_dto: MaintenanceGetIntegrityReportDto): Promise { + return { + items: await this.db + .selectFrom('integrity_report') + .select(['type', 'path']) + .orderBy('createdAt', 'desc') + .execute(), + }; + } + deleteById(id: string) { return this.db.deleteFrom('integrity_report').where('id', '=', id).execute(); } diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index b876004385..fcc7589642 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -33,7 +33,7 @@ async function* chunk(generator: AsyncIterableIterator, n: number) { } } - if (chunk.length) { + if (chunk.length > 0) { yield chunk; } } @@ -83,17 +83,17 @@ export class IntegrityService extends BaseService { // debug: run on boot setImmediate(() => { - this.jobRepository.queue({ + void this.jobRepository.queue({ name: JobName.IntegrityOrphanedFilesQueueAll, data: {}, }); - this.jobRepository.queue({ + void this.jobRepository.queue({ name: JobName.IntegrityMissingFilesQueueAll, data: {}, }); - this.jobRepository.queue({ + void this.jobRepository.queue({ name: JobName.IntegrityChecksumFiles, data: {}, }); @@ -101,7 +101,7 @@ export class IntegrityService extends BaseService { } @OnEvent({ name: 'ConfigUpdate', server: true }) - async onConfigUpdate({ + onConfigUpdate({ newConfig: { integrityChecks: { orphanedFiles, missingFiles, checksumFiles }, }, @@ -237,9 +237,9 @@ export class IntegrityService extends BaseService { ), ); - const reportIds = results.filter((reportId) => reportId) as string[]; + const reportIds = results.filter(Boolean) as string[]; - if (reportIds.length) { + if (reportIds.length > 0) { await this.integrityReportRepository.deleteByIds(reportIds); } @@ -285,12 +285,12 @@ export class IntegrityService extends BaseService { .filter(({ exists, reportId }) => exists && reportId) .map(({ reportId }) => reportId!); - if (outdatedReports.length) { + if (outdatedReports.length > 0) { await this.integrityReportRepository.deleteByIds(outdatedReports); } const missingFiles = results.filter(({ exists }) => !exists); - if (missingFiles.length) { + if (missingFiles.length > 0) { await this.integrityReportRepository.create( missingFiles.map(({ path }) => ({ type: IntegrityReportType.MissingFile, @@ -306,7 +306,7 @@ export class IntegrityService extends BaseService { @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.BackgroundTask }) async handleChecksumFiles(): Promise { const timeLimit = 60 * 60 * 1000; // 1000; - const percentageLimit = 1.0; // 0.25; + const percentageLimit = 1; // 0.25; this.logger.log( `Checking file checksums... (will run for up to ${(timeLimit / (60 * 60 * 1000)).toFixed(2)} hours or until ${(percentageLimit * 100).toFixed(2)}% of assets are processed)`, @@ -390,7 +390,7 @@ export class IntegrityService extends BaseService { } } while (endMarker); - this.systemMetadataRepository.set(SystemMetadataKey.IntegrityChecksumCheckpoint, { + await this.systemMetadataRepository.set(SystemMetadataKey.IntegrityChecksumCheckpoint, { date: lastCreatedAt?.toISOString(), }); diff --git a/server/src/services/maintenance.service.ts b/server/src/services/maintenance.service.ts index e6808300bc..7438acfc1d 100644 --- a/server/src/services/maintenance.service.ts +++ b/server/src/services/maintenance.service.ts @@ -1,6 +1,10 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from 'src/decorators'; -import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto'; +import { + MaintenanceAuthDto, + MaintenanceGetIntegrityReportDto, + MaintenanceIntegrityReportResponseDto, +} from 'src/dtos/maintenance.dto'; import { SystemMetadataKey } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { MaintenanceModeState } from 'src/types'; @@ -50,4 +54,8 @@ export class MaintenanceService extends BaseService { return await createMaintenanceLoginUrl(baseUrl, auth, secret); } + + getIntegrityReport(dto: MaintenanceGetIntegrityReportDto): Promise { + return this.integrityReportRepository.getIntegrityReport(dto); + } } diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 8f5ed468de..7fe13ad84f 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -85,7 +85,7 @@ const updatedConfig = Object.freeze({ enabled: true, cronExpression: '0 03 * * *', timeLimit: 60 * 60 * 1000, - percentageLimit: 1.0, + percentageLimit: 1, }, }, logging: { From d3abed341448849ca5300768a4c26acb09e5360c Mon Sep 17 00:00:00 2001 From: izzy Date: Thu, 27 Nov 2025 17:53:20 +0000 Subject: [PATCH 17/50] feat: view integrity report in maintenance page (cherry picked) --- mobile/openapi/README.md | 3 + mobile/openapi/lib/api.dart | 2 + .../lib/api/maintenance_admin_api.dart | 48 +++++ mobile/openapi/lib/api_client.dart | 4 + .../maintenance_integrity_report_dto.dart | 192 ++++++++++++++++++ ...tenance_integrity_report_response_dto.dart | 99 +++++++++ open-api/immich-openapi-specs.json | 4 + open-api/typescript-sdk/src/fetch-client.ts | 24 +++ server/src/dtos/maintenance.dto.ts | 1 + .../integrity-report.repository.ts | 2 +- web/src/lib/constants.ts | 1 + web/src/lib/sidebars/AdminSidebar.svelte | 1 + web/src/routes/admin/maintenance/+page.svelte | 92 +++++++++ web/src/routes/admin/maintenance/+page.ts | 17 ++ 14 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 mobile/openapi/lib/model/maintenance_integrity_report_dto.dart create mode 100644 mobile/openapi/lib/model/maintenance_integrity_report_response_dto.dart create mode 100644 web/src/routes/admin/maintenance/+page.svelte create mode 100644 web/src/routes/admin/maintenance/+page.ts diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 391c7c3759..3b424e8d79 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -161,6 +161,7 @@ Class | Method | HTTP request | Description *LibrariesApi* | [**scanLibrary**](doc//LibrariesApi.md#scanlibrary) | **POST** /libraries/{id}/scan | Scan a library *LibrariesApi* | [**updateLibrary**](doc//LibrariesApi.md#updatelibrary) | **PUT** /libraries/{id} | Update a library *LibrariesApi* | [**validate**](doc//LibrariesApi.md#validate) | **POST** /libraries/{id}/validate | Validate library settings +*MaintenanceAdminApi* | [**getIntegrityReport**](doc//MaintenanceAdminApi.md#getintegrityreport) | **GET** /admin/maintenance | Get integrity report *MaintenanceAdminApi* | [**maintenanceLogin**](doc//MaintenanceAdminApi.md#maintenancelogin) | **POST** /admin/maintenance/login | Log into maintenance mode *MaintenanceAdminApi* | [**setMaintenanceMode**](doc//MaintenanceAdminApi.md#setmaintenancemode) | **POST** /admin/maintenance | Set maintenance mode *MapApi* | [**getMapMarkers**](doc//MapApi.md#getmapmarkers) | **GET** /map/markers | Retrieve map markers @@ -416,6 +417,8 @@ Class | Method | HTTP request | Description - [MachineLearningAvailabilityChecksDto](doc//MachineLearningAvailabilityChecksDto.md) - [MaintenanceAction](doc//MaintenanceAction.md) - [MaintenanceAuthDto](doc//MaintenanceAuthDto.md) + - [MaintenanceIntegrityReportDto](doc//MaintenanceIntegrityReportDto.md) + - [MaintenanceIntegrityReportResponseDto](doc//MaintenanceIntegrityReportResponseDto.md) - [MaintenanceLoginDto](doc//MaintenanceLoginDto.md) - [ManualJobName](doc//ManualJobName.md) - [MapMarkerResponseDto](doc//MapMarkerResponseDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 641000daf7..ddacd27a3b 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -168,6 +168,8 @@ part 'model/logout_response_dto.dart'; part 'model/machine_learning_availability_checks_dto.dart'; part 'model/maintenance_action.dart'; part 'model/maintenance_auth_dto.dart'; +part 'model/maintenance_integrity_report_dto.dart'; +part 'model/maintenance_integrity_report_response_dto.dart'; part 'model/maintenance_login_dto.dart'; part 'model/manual_job_name.dart'; part 'model/map_marker_response_dto.dart'; diff --git a/mobile/openapi/lib/api/maintenance_admin_api.dart b/mobile/openapi/lib/api/maintenance_admin_api.dart index 7e46f96c6e..c0a9aa862f 100644 --- a/mobile/openapi/lib/api/maintenance_admin_api.dart +++ b/mobile/openapi/lib/api/maintenance_admin_api.dart @@ -16,6 +16,54 @@ class MaintenanceAdminApi { final ApiClient apiClient; + /// Get integrity report + /// + /// ... + /// + /// Note: This method returns the HTTP [Response]. + Future getIntegrityReportWithHttpInfo() async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/maintenance'; + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Get integrity report + /// + /// ... + Future getIntegrityReport() async { + final response = await getIntegrityReportWithHttpInfo(); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'MaintenanceIntegrityReportResponseDto',) as MaintenanceIntegrityReportResponseDto; + + } + return null; + } + /// Log into maintenance mode /// /// Login with maintenance token or cookie to receive current information and perform further actions. diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index ae2fa85a93..c522db3023 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -384,6 +384,10 @@ class ApiClient { return MaintenanceActionTypeTransformer().decode(value); case 'MaintenanceAuthDto': return MaintenanceAuthDto.fromJson(value); + case 'MaintenanceIntegrityReportDto': + return MaintenanceIntegrityReportDto.fromJson(value); + case 'MaintenanceIntegrityReportResponseDto': + return MaintenanceIntegrityReportResponseDto.fromJson(value); case 'MaintenanceLoginDto': return MaintenanceLoginDto.fromJson(value); case 'ManualJobName': diff --git a/mobile/openapi/lib/model/maintenance_integrity_report_dto.dart b/mobile/openapi/lib/model/maintenance_integrity_report_dto.dart new file mode 100644 index 0000000000..c5e4baad86 --- /dev/null +++ b/mobile/openapi/lib/model/maintenance_integrity_report_dto.dart @@ -0,0 +1,192 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class MaintenanceIntegrityReportDto { + /// Returns a new [MaintenanceIntegrityReportDto] instance. + MaintenanceIntegrityReportDto({ + required this.id, + required this.path, + required this.type, + }); + + String id; + + String path; + + MaintenanceIntegrityReportDtoTypeEnum type; + + @override + bool operator ==(Object other) => identical(this, other) || other is MaintenanceIntegrityReportDto && + other.id == id && + other.path == path && + other.type == type; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (id.hashCode) + + (path.hashCode) + + (type.hashCode); + + @override + String toString() => 'MaintenanceIntegrityReportDto[id=$id, path=$path, type=$type]'; + + Map toJson() { + final json = {}; + json[r'id'] = this.id; + json[r'path'] = this.path; + json[r'type'] = this.type; + return json; + } + + /// Returns a new [MaintenanceIntegrityReportDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static MaintenanceIntegrityReportDto? fromJson(dynamic value) { + upgradeDto(value, "MaintenanceIntegrityReportDto"); + if (value is Map) { + final json = value.cast(); + + return MaintenanceIntegrityReportDto( + id: mapValueOfType(json, r'id')!, + path: mapValueOfType(json, r'path')!, + type: MaintenanceIntegrityReportDtoTypeEnum.fromJson(json[r'type'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = MaintenanceIntegrityReportDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = MaintenanceIntegrityReportDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of MaintenanceIntegrityReportDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = MaintenanceIntegrityReportDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'id', + 'path', + 'type', + }; +} + + +class MaintenanceIntegrityReportDtoTypeEnum { + /// Instantiate a new enum with the provided [value]. + const MaintenanceIntegrityReportDtoTypeEnum._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const orphanFile = MaintenanceIntegrityReportDtoTypeEnum._(r'orphan_file'); + static const missingFile = MaintenanceIntegrityReportDtoTypeEnum._(r'missing_file'); + static const checksumMismatch = MaintenanceIntegrityReportDtoTypeEnum._(r'checksum_mismatch'); + + /// List of all possible values in this [enum][MaintenanceIntegrityReportDtoTypeEnum]. + static const values = [ + orphanFile, + missingFile, + checksumMismatch, + ]; + + static MaintenanceIntegrityReportDtoTypeEnum? fromJson(dynamic value) => MaintenanceIntegrityReportDtoTypeEnumTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = MaintenanceIntegrityReportDtoTypeEnum.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [MaintenanceIntegrityReportDtoTypeEnum] to String, +/// and [decode] dynamic data back to [MaintenanceIntegrityReportDtoTypeEnum]. +class MaintenanceIntegrityReportDtoTypeEnumTypeTransformer { + factory MaintenanceIntegrityReportDtoTypeEnumTypeTransformer() => _instance ??= const MaintenanceIntegrityReportDtoTypeEnumTypeTransformer._(); + + const MaintenanceIntegrityReportDtoTypeEnumTypeTransformer._(); + + String encode(MaintenanceIntegrityReportDtoTypeEnum data) => data.value; + + /// Decodes a [dynamic value][data] to a MaintenanceIntegrityReportDtoTypeEnum. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + MaintenanceIntegrityReportDtoTypeEnum? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'orphan_file': return MaintenanceIntegrityReportDtoTypeEnum.orphanFile; + case r'missing_file': return MaintenanceIntegrityReportDtoTypeEnum.missingFile; + case r'checksum_mismatch': return MaintenanceIntegrityReportDtoTypeEnum.checksumMismatch; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [MaintenanceIntegrityReportDtoTypeEnumTypeTransformer] instance. + static MaintenanceIntegrityReportDtoTypeEnumTypeTransformer? _instance; +} + + diff --git a/mobile/openapi/lib/model/maintenance_integrity_report_response_dto.dart b/mobile/openapi/lib/model/maintenance_integrity_report_response_dto.dart new file mode 100644 index 0000000000..7d76a05db6 --- /dev/null +++ b/mobile/openapi/lib/model/maintenance_integrity_report_response_dto.dart @@ -0,0 +1,99 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class MaintenanceIntegrityReportResponseDto { + /// Returns a new [MaintenanceIntegrityReportResponseDto] instance. + MaintenanceIntegrityReportResponseDto({ + this.items = const [], + }); + + List items; + + @override + bool operator ==(Object other) => identical(this, other) || other is MaintenanceIntegrityReportResponseDto && + _deepEquality.equals(other.items, items); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (items.hashCode); + + @override + String toString() => 'MaintenanceIntegrityReportResponseDto[items=$items]'; + + Map toJson() { + final json = {}; + json[r'items'] = this.items; + return json; + } + + /// Returns a new [MaintenanceIntegrityReportResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static MaintenanceIntegrityReportResponseDto? fromJson(dynamic value) { + upgradeDto(value, "MaintenanceIntegrityReportResponseDto"); + if (value is Map) { + final json = value.cast(); + + return MaintenanceIntegrityReportResponseDto( + items: MaintenanceIntegrityReportDto.listFromJson(json[r'items']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = MaintenanceIntegrityReportResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = MaintenanceIntegrityReportResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of MaintenanceIntegrityReportResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = MaintenanceIntegrityReportResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'items', + }; +} + diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 0d87727e8a..8e39950f20 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -16967,6 +16967,9 @@ }, "MaintenanceIntegrityReportDto": { "properties": { + "id": { + "type": "string" + }, "path": { "type": "string" }, @@ -16980,6 +16983,7 @@ } }, "required": [ + "id", "path", "type" ], diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index b33276a90b..d4802f2abc 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -40,6 +40,14 @@ export type ActivityStatisticsResponseDto = { comments: number; likes: number; }; +export type MaintenanceIntegrityReportDto = { + id: string; + path: string; + "type": Type; +}; +export type MaintenanceIntegrityReportResponseDto = { + items: MaintenanceIntegrityReportDto[]; +}; export type SetMaintenanceModeDto = { action: MaintenanceAction; }; @@ -1866,6 +1874,17 @@ export function unlinkAllOAuthAccountsAdmin(opts?: Oazapfts.RequestOpts) { method: "POST" })); } +/** + * Get integrity report + */ +export function getIntegrityReport(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: MaintenanceIntegrityReportResponseDto; + }>("/admin/maintenance", { + ...opts + })); +} /** * Set maintenance mode */ @@ -5154,6 +5173,11 @@ export enum UserAvatarColor { Gray = "gray", Amber = "amber" } +export enum Type { + OrphanFile = "orphan_file", + MissingFile = "missing_file", + ChecksumMismatch = "checksum_mismatch" +} export enum MaintenanceAction { Start = "start", End = "end" diff --git a/server/src/dtos/maintenance.dto.ts b/server/src/dtos/maintenance.dto.ts index 6d7f15f55c..444b843e92 100644 --- a/server/src/dtos/maintenance.dto.ts +++ b/server/src/dtos/maintenance.dto.ts @@ -26,6 +26,7 @@ export class MaintenanceGetIntegrityReportDto { } class MaintenanceIntegrityReportDto { + id!: string; @IsEnum(IntegrityReportType) type!: IntegrityReportType; path!: string; diff --git a/server/src/repositories/integrity-report.repository.ts b/server/src/repositories/integrity-report.repository.ts index af36f051de..e3b9af6457 100644 --- a/server/src/repositories/integrity-report.repository.ts +++ b/server/src/repositories/integrity-report.repository.ts @@ -22,7 +22,7 @@ export class IntegrityReportRepository { return { items: await this.db .selectFrom('integrity_report') - .select(['type', 'path']) + .select(['id', 'type', 'path']) .orderBy('createdAt', 'desc') .execute(), }; diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts index 4cd238cb52..7b1ae7ebb3 100644 --- a/web/src/lib/constants.ts +++ b/web/src/lib/constants.ts @@ -22,6 +22,7 @@ export enum AppRoute { ADMIN_USERS = '/admin/users', ADMIN_LIBRARY_MANAGEMENT = '/admin/library-management', ADMIN_SETTINGS = '/admin/system-settings', + ADMIN_MAINTENANCE_SETTINGS = '/admin/maintenance', ADMIN_STATS = '/admin/server-status', ADMIN_JOBS = '/admin/jobs-status', ADMIN_REPAIR = '/admin/repair', diff --git a/web/src/lib/sidebars/AdminSidebar.svelte b/web/src/lib/sidebars/AdminSidebar.svelte index 2fecaebf49..2418aa393e 100644 --- a/web/src/lib/sidebars/AdminSidebar.svelte +++ b/web/src/lib/sidebars/AdminSidebar.svelte @@ -11,6 +11,7 @@ + diff --git a/web/src/routes/admin/maintenance/+page.svelte b/web/src/routes/admin/maintenance/+page.svelte new file mode 100644 index 0000000000..12a0e4067d --- /dev/null +++ b/web/src/routes/admin/maintenance/+page.svelte @@ -0,0 +1,92 @@ + + + + {#snippet buttons()} + + + + {/snippet} + +
+
+ + + + + + + + + + + + {#each data.integrityReport.items as { id, type, path } (id)} + + + + + + {/each} + +
ReasonFile
+ {#if type === 'orphan_file'} + Orphaned File + {:else if type === 'missing_file'} + Missing File + {:else if type === 'checksum_mismatch'} + Checksum Mismatch + {/if} + {path} +
+ + +
+
+ diff --git a/web/src/routes/admin/maintenance/+page.ts b/web/src/routes/admin/maintenance/+page.ts new file mode 100644 index 0000000000..65ecbf9a77 --- /dev/null +++ b/web/src/routes/admin/maintenance/+page.ts @@ -0,0 +1,17 @@ +import { authenticate } from '$lib/utils/auth'; +import { getFormatter } from '$lib/utils/i18n'; +import { getIntegrityReport } from '@immich/sdk'; +import type { PageLoad } from './$types'; + +export const load = (async ({ url }) => { + await authenticate(url, { admin: true }); + const integrityReport = await getIntegrityReport(); + const $t = await getFormatter(); + + return { + integrityReport, + meta: { + title: $t('admin.system_settings'), + }, + }; +}) satisfies PageLoad; From ca358f4dae04ce401c16d54b9edf18446663e95a Mon Sep 17 00:00:00 2001 From: izzy Date: Fri, 28 Nov 2025 11:40:53 +0000 Subject: [PATCH 18/50] feat: sub-pages for integrity reports --- i18n/en.json | 4 + mobile/openapi/README.md | 6 +- mobile/openapi/lib/api.dart | 3 + .../lib/api/maintenance_admin_api.dart | 70 +++++- mobile/openapi/lib/api_client.dart | 6 + mobile/openapi/lib/api_helper.dart | 3 + .../lib/model/integrity_report_type.dart | 88 ++++++++ .../maintenance_get_integrity_report_dto.dart | 99 +++++++++ .../maintenance_integrity_report_dto.dart | 81 +------ ...integrity_report_summary_response_dto.dart | 115 ++++++++++ open-api/immich-openapi-specs.json | 202 +++++++++++++----- open-api/typescript-sdk/src/fetch-client.ts | 61 ++++-- .../src/controllers/maintenance.controller.ts | 18 +- server/src/dtos/maintenance.dto.ts | 16 +- .../integrity-report.repository.ts | 34 ++- server/src/services/maintenance.service.ts | 5 + .../ServerStatisticsCard.svelte | 12 +- web/src/lib/constants.ts | 1 + web/src/lib/sidebars/AdminSidebar.svelte | 4 +- web/src/routes/admin/maintenance/+page.svelte | 32 ++- web/src/routes/admin/maintenance/+page.ts | 6 +- .../integrity-report/[type]/+page.svelte | 75 +++++++ .../integrity-report/[type]/+page.ts | 23 ++ 23 files changed, 785 insertions(+), 179 deletions(-) create mode 100644 mobile/openapi/lib/model/integrity_report_type.dart create mode 100644 mobile/openapi/lib/model/maintenance_get_integrity_report_dto.dart create mode 100644 mobile/openapi/lib/model/maintenance_integrity_report_summary_response_dto.dart create mode 100644 web/src/routes/admin/maintenance/integrity-report/[type]/+page.svelte create mode 100644 web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts diff --git a/i18n/en.json b/i18n/en.json index 210e05459d..8870b31a9a 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -181,6 +181,10 @@ "maintenance_settings_description": "Put Immich into maintenance mode.", "maintenance_start": "Start maintenance mode", "maintenance_start_error": "Failed to start maintenance mode.", + "maintenance_integrity_report": "Integrity Report", + "maintenance_integrity_orphan_file": "Orphan Files", + "maintenance_integrity_missing_file": "Missing Files", + "maintenance_integrity_checksum_mismatch": "Checksum Mismatch", "manage_concurrency": "Manage Concurrency", "manage_log_settings": "Manage log settings", "map_dark_style": "Dark style", diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 3b424e8d79..d9eb463717 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -161,7 +161,8 @@ Class | Method | HTTP request | Description *LibrariesApi* | [**scanLibrary**](doc//LibrariesApi.md#scanlibrary) | **POST** /libraries/{id}/scan | Scan a library *LibrariesApi* | [**updateLibrary**](doc//LibrariesApi.md#updatelibrary) | **PUT** /libraries/{id} | Update a library *LibrariesApi* | [**validate**](doc//LibrariesApi.md#validate) | **POST** /libraries/{id}/validate | Validate library settings -*MaintenanceAdminApi* | [**getIntegrityReport**](doc//MaintenanceAdminApi.md#getintegrityreport) | **GET** /admin/maintenance | Get integrity report +*MaintenanceAdminApi* | [**getIntegrityReport**](doc//MaintenanceAdminApi.md#getintegrityreport) | **POST** /admin/maintenance/integrity/report | Get integrity report by type +*MaintenanceAdminApi* | [**getIntegrityReportSummary**](doc//MaintenanceAdminApi.md#getintegrityreportsummary) | **GET** /admin/maintenance/integrity/summary | Get integrity report summary *MaintenanceAdminApi* | [**maintenanceLogin**](doc//MaintenanceAdminApi.md#maintenancelogin) | **POST** /admin/maintenance/login | Log into maintenance mode *MaintenanceAdminApi* | [**setMaintenanceMode**](doc//MaintenanceAdminApi.md#setmaintenancemode) | **POST** /admin/maintenance | Set maintenance mode *MapApi* | [**getMapMarkers**](doc//MapApi.md#getmapmarkers) | **GET** /map/markers | Retrieve map markers @@ -403,6 +404,7 @@ Class | Method | HTTP request | Description - [FoldersResponse](doc//FoldersResponse.md) - [FoldersUpdate](doc//FoldersUpdate.md) - [ImageFormat](doc//ImageFormat.md) + - [IntegrityReportType](doc//IntegrityReportType.md) - [JobCreateDto](doc//JobCreateDto.md) - [JobName](doc//JobName.md) - [JobSettingsDto](doc//JobSettingsDto.md) @@ -417,8 +419,10 @@ Class | Method | HTTP request | Description - [MachineLearningAvailabilityChecksDto](doc//MachineLearningAvailabilityChecksDto.md) - [MaintenanceAction](doc//MaintenanceAction.md) - [MaintenanceAuthDto](doc//MaintenanceAuthDto.md) + - [MaintenanceGetIntegrityReportDto](doc//MaintenanceGetIntegrityReportDto.md) - [MaintenanceIntegrityReportDto](doc//MaintenanceIntegrityReportDto.md) - [MaintenanceIntegrityReportResponseDto](doc//MaintenanceIntegrityReportResponseDto.md) + - [MaintenanceIntegrityReportSummaryResponseDto](doc//MaintenanceIntegrityReportSummaryResponseDto.md) - [MaintenanceLoginDto](doc//MaintenanceLoginDto.md) - [ManualJobName](doc//ManualJobName.md) - [MapMarkerResponseDto](doc//MapMarkerResponseDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index ddacd27a3b..3ee5e5c886 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -154,6 +154,7 @@ part 'model/facial_recognition_config.dart'; part 'model/folders_response.dart'; part 'model/folders_update.dart'; part 'model/image_format.dart'; +part 'model/integrity_report_type.dart'; part 'model/job_create_dto.dart'; part 'model/job_name.dart'; part 'model/job_settings_dto.dart'; @@ -168,8 +169,10 @@ part 'model/logout_response_dto.dart'; part 'model/machine_learning_availability_checks_dto.dart'; part 'model/maintenance_action.dart'; part 'model/maintenance_auth_dto.dart'; +part 'model/maintenance_get_integrity_report_dto.dart'; part 'model/maintenance_integrity_report_dto.dart'; part 'model/maintenance_integrity_report_response_dto.dart'; +part 'model/maintenance_integrity_report_summary_response_dto.dart'; part 'model/maintenance_login_dto.dart'; part 'model/manual_job_name.dart'; part 'model/map_marker_response_dto.dart'; diff --git a/mobile/openapi/lib/api/maintenance_admin_api.dart b/mobile/openapi/lib/api/maintenance_admin_api.dart index c0a9aa862f..6023b36fc6 100644 --- a/mobile/openapi/lib/api/maintenance_admin_api.dart +++ b/mobile/openapi/lib/api/maintenance_admin_api.dart @@ -16,14 +16,70 @@ class MaintenanceAdminApi { final ApiClient apiClient; - /// Get integrity report + /// Get integrity report by type /// /// ... /// /// Note: This method returns the HTTP [Response]. - Future getIntegrityReportWithHttpInfo() async { + /// + /// Parameters: + /// + /// * [MaintenanceGetIntegrityReportDto] maintenanceGetIntegrityReportDto (required): + Future getIntegrityReportWithHttpInfo(MaintenanceGetIntegrityReportDto maintenanceGetIntegrityReportDto,) async { // ignore: prefer_const_declarations - final apiPath = r'/admin/maintenance'; + final apiPath = r'/admin/maintenance/integrity/report'; + + // ignore: prefer_final_locals + Object? postBody = maintenanceGetIntegrityReportDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + apiPath, + 'POST', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Get integrity report by type + /// + /// ... + /// + /// Parameters: + /// + /// * [MaintenanceGetIntegrityReportDto] maintenanceGetIntegrityReportDto (required): + Future getIntegrityReport(MaintenanceGetIntegrityReportDto maintenanceGetIntegrityReportDto,) async { + final response = await getIntegrityReportWithHttpInfo(maintenanceGetIntegrityReportDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'MaintenanceIntegrityReportResponseDto',) as MaintenanceIntegrityReportResponseDto; + + } + return null; + } + + /// Get integrity report summary + /// + /// ... + /// + /// Note: This method returns the HTTP [Response]. + Future getIntegrityReportSummaryWithHttpInfo() async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/maintenance/integrity/summary'; // ignore: prefer_final_locals Object? postBody; @@ -46,11 +102,11 @@ class MaintenanceAdminApi { ); } - /// Get integrity report + /// Get integrity report summary /// /// ... - Future getIntegrityReport() async { - final response = await getIntegrityReportWithHttpInfo(); + Future getIntegrityReportSummary() async { + final response = await getIntegrityReportSummaryWithHttpInfo(); if (response.statusCode >= HttpStatus.badRequest) { throw ApiException(response.statusCode, await _decodeBodyBytes(response)); } @@ -58,7 +114,7 @@ class MaintenanceAdminApi { // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" // FormatException when trying to decode an empty string. if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { - return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'MaintenanceIntegrityReportResponseDto',) as MaintenanceIntegrityReportResponseDto; + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'MaintenanceIntegrityReportSummaryResponseDto',) as MaintenanceIntegrityReportSummaryResponseDto; } return null; diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index c522db3023..889c18bde0 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -356,6 +356,8 @@ class ApiClient { return FoldersUpdate.fromJson(value); case 'ImageFormat': return ImageFormatTypeTransformer().decode(value); + case 'IntegrityReportType': + return IntegrityReportTypeTypeTransformer().decode(value); case 'JobCreateDto': return JobCreateDto.fromJson(value); case 'JobName': @@ -384,10 +386,14 @@ class ApiClient { return MaintenanceActionTypeTransformer().decode(value); case 'MaintenanceAuthDto': return MaintenanceAuthDto.fromJson(value); + case 'MaintenanceGetIntegrityReportDto': + return MaintenanceGetIntegrityReportDto.fromJson(value); case 'MaintenanceIntegrityReportDto': return MaintenanceIntegrityReportDto.fromJson(value); case 'MaintenanceIntegrityReportResponseDto': return MaintenanceIntegrityReportResponseDto.fromJson(value); + case 'MaintenanceIntegrityReportSummaryResponseDto': + return MaintenanceIntegrityReportSummaryResponseDto.fromJson(value); case 'MaintenanceLoginDto': return MaintenanceLoginDto.fromJson(value); case 'ManualJobName': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index 2c97eeb314..85477a4f6c 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -94,6 +94,9 @@ String parameterToString(dynamic value) { if (value is ImageFormat) { return ImageFormatTypeTransformer().encode(value).toString(); } + if (value is IntegrityReportType) { + return IntegrityReportTypeTypeTransformer().encode(value).toString(); + } if (value is JobName) { return JobNameTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/integrity_report_type.dart b/mobile/openapi/lib/model/integrity_report_type.dart new file mode 100644 index 0000000000..f027cd6f5a --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report_type.dart @@ -0,0 +1,88 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class IntegrityReportType { + /// Instantiate a new enum with the provided [value]. + const IntegrityReportType._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const orphanFile = IntegrityReportType._(r'orphan_file'); + static const missingFile = IntegrityReportType._(r'missing_file'); + static const checksumMismatch = IntegrityReportType._(r'checksum_mismatch'); + + /// List of all possible values in this [enum][IntegrityReportType]. + static const values = [ + orphanFile, + missingFile, + checksumMismatch, + ]; + + static IntegrityReportType? fromJson(dynamic value) => IntegrityReportTypeTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReportType.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [IntegrityReportType] to String, +/// and [decode] dynamic data back to [IntegrityReportType]. +class IntegrityReportTypeTypeTransformer { + factory IntegrityReportTypeTypeTransformer() => _instance ??= const IntegrityReportTypeTypeTransformer._(); + + const IntegrityReportTypeTypeTransformer._(); + + String encode(IntegrityReportType data) => data.value; + + /// Decodes a [dynamic value][data] to a IntegrityReportType. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + IntegrityReportType? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'orphan_file': return IntegrityReportType.orphanFile; + case r'missing_file': return IntegrityReportType.missingFile; + case r'checksum_mismatch': return IntegrityReportType.checksumMismatch; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [IntegrityReportTypeTypeTransformer] instance. + static IntegrityReportTypeTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/maintenance_get_integrity_report_dto.dart b/mobile/openapi/lib/model/maintenance_get_integrity_report_dto.dart new file mode 100644 index 0000000000..691daa5ea7 --- /dev/null +++ b/mobile/openapi/lib/model/maintenance_get_integrity_report_dto.dart @@ -0,0 +1,99 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class MaintenanceGetIntegrityReportDto { + /// Returns a new [MaintenanceGetIntegrityReportDto] instance. + MaintenanceGetIntegrityReportDto({ + required this.type, + }); + + IntegrityReportType type; + + @override + bool operator ==(Object other) => identical(this, other) || other is MaintenanceGetIntegrityReportDto && + other.type == type; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (type.hashCode); + + @override + String toString() => 'MaintenanceGetIntegrityReportDto[type=$type]'; + + Map toJson() { + final json = {}; + json[r'type'] = this.type; + return json; + } + + /// Returns a new [MaintenanceGetIntegrityReportDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static MaintenanceGetIntegrityReportDto? fromJson(dynamic value) { + upgradeDto(value, "MaintenanceGetIntegrityReportDto"); + if (value is Map) { + final json = value.cast(); + + return MaintenanceGetIntegrityReportDto( + type: IntegrityReportType.fromJson(json[r'type'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = MaintenanceGetIntegrityReportDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = MaintenanceGetIntegrityReportDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of MaintenanceGetIntegrityReportDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = MaintenanceGetIntegrityReportDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'type', + }; +} + diff --git a/mobile/openapi/lib/model/maintenance_integrity_report_dto.dart b/mobile/openapi/lib/model/maintenance_integrity_report_dto.dart index c5e4baad86..777a6f0534 100644 --- a/mobile/openapi/lib/model/maintenance_integrity_report_dto.dart +++ b/mobile/openapi/lib/model/maintenance_integrity_report_dto.dart @@ -22,7 +22,7 @@ class MaintenanceIntegrityReportDto { String path; - MaintenanceIntegrityReportDtoTypeEnum type; + IntegrityReportType type; @override bool operator ==(Object other) => identical(this, other) || other is MaintenanceIntegrityReportDto && @@ -59,7 +59,7 @@ class MaintenanceIntegrityReportDto { return MaintenanceIntegrityReportDto( id: mapValueOfType(json, r'id')!, path: mapValueOfType(json, r'path')!, - type: MaintenanceIntegrityReportDtoTypeEnum.fromJson(json[r'type'])!, + type: IntegrityReportType.fromJson(json[r'type'])!, ); } return null; @@ -113,80 +113,3 @@ class MaintenanceIntegrityReportDto { }; } - -class MaintenanceIntegrityReportDtoTypeEnum { - /// Instantiate a new enum with the provided [value]. - const MaintenanceIntegrityReportDtoTypeEnum._(this.value); - - /// The underlying value of this enum member. - final String value; - - @override - String toString() => value; - - String toJson() => value; - - static const orphanFile = MaintenanceIntegrityReportDtoTypeEnum._(r'orphan_file'); - static const missingFile = MaintenanceIntegrityReportDtoTypeEnum._(r'missing_file'); - static const checksumMismatch = MaintenanceIntegrityReportDtoTypeEnum._(r'checksum_mismatch'); - - /// List of all possible values in this [enum][MaintenanceIntegrityReportDtoTypeEnum]. - static const values = [ - orphanFile, - missingFile, - checksumMismatch, - ]; - - static MaintenanceIntegrityReportDtoTypeEnum? fromJson(dynamic value) => MaintenanceIntegrityReportDtoTypeEnumTypeTransformer().decode(value); - - static List listFromJson(dynamic json, {bool growable = false,}) { - final result = []; - if (json is List && json.isNotEmpty) { - for (final row in json) { - final value = MaintenanceIntegrityReportDtoTypeEnum.fromJson(row); - if (value != null) { - result.add(value); - } - } - } - return result.toList(growable: growable); - } -} - -/// Transformation class that can [encode] an instance of [MaintenanceIntegrityReportDtoTypeEnum] to String, -/// and [decode] dynamic data back to [MaintenanceIntegrityReportDtoTypeEnum]. -class MaintenanceIntegrityReportDtoTypeEnumTypeTransformer { - factory MaintenanceIntegrityReportDtoTypeEnumTypeTransformer() => _instance ??= const MaintenanceIntegrityReportDtoTypeEnumTypeTransformer._(); - - const MaintenanceIntegrityReportDtoTypeEnumTypeTransformer._(); - - String encode(MaintenanceIntegrityReportDtoTypeEnum data) => data.value; - - /// Decodes a [dynamic value][data] to a MaintenanceIntegrityReportDtoTypeEnum. - /// - /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, - /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] - /// cannot be decoded successfully, then an [UnimplementedError] is thrown. - /// - /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, - /// and users are still using an old app with the old code. - MaintenanceIntegrityReportDtoTypeEnum? decode(dynamic data, {bool allowNull = true}) { - if (data != null) { - switch (data) { - case r'orphan_file': return MaintenanceIntegrityReportDtoTypeEnum.orphanFile; - case r'missing_file': return MaintenanceIntegrityReportDtoTypeEnum.missingFile; - case r'checksum_mismatch': return MaintenanceIntegrityReportDtoTypeEnum.checksumMismatch; - default: - if (!allowNull) { - throw ArgumentError('Unknown enum value to decode: $data'); - } - } - } - return null; - } - - /// Singleton [MaintenanceIntegrityReportDtoTypeEnumTypeTransformer] instance. - static MaintenanceIntegrityReportDtoTypeEnumTypeTransformer? _instance; -} - - diff --git a/mobile/openapi/lib/model/maintenance_integrity_report_summary_response_dto.dart b/mobile/openapi/lib/model/maintenance_integrity_report_summary_response_dto.dart new file mode 100644 index 0000000000..fbf53b9436 --- /dev/null +++ b/mobile/openapi/lib/model/maintenance_integrity_report_summary_response_dto.dart @@ -0,0 +1,115 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class MaintenanceIntegrityReportSummaryResponseDto { + /// Returns a new [MaintenanceIntegrityReportSummaryResponseDto] instance. + MaintenanceIntegrityReportSummaryResponseDto({ + required this.checksumMismatch, + required this.missingFile, + required this.orphanFile, + }); + + int checksumMismatch; + + int missingFile; + + int orphanFile; + + @override + bool operator ==(Object other) => identical(this, other) || other is MaintenanceIntegrityReportSummaryResponseDto && + other.checksumMismatch == checksumMismatch && + other.missingFile == missingFile && + other.orphanFile == orphanFile; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (checksumMismatch.hashCode) + + (missingFile.hashCode) + + (orphanFile.hashCode); + + @override + String toString() => 'MaintenanceIntegrityReportSummaryResponseDto[checksumMismatch=$checksumMismatch, missingFile=$missingFile, orphanFile=$orphanFile]'; + + Map toJson() { + final json = {}; + json[r'checksum_mismatch'] = this.checksumMismatch; + json[r'missing_file'] = this.missingFile; + json[r'orphan_file'] = this.orphanFile; + return json; + } + + /// Returns a new [MaintenanceIntegrityReportSummaryResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static MaintenanceIntegrityReportSummaryResponseDto? fromJson(dynamic value) { + upgradeDto(value, "MaintenanceIntegrityReportSummaryResponseDto"); + if (value is Map) { + final json = value.cast(); + + return MaintenanceIntegrityReportSummaryResponseDto( + checksumMismatch: mapValueOfType(json, r'checksum_mismatch')!, + missingFile: mapValueOfType(json, r'missing_file')!, + orphanFile: mapValueOfType(json, r'orphan_file')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = MaintenanceIntegrityReportSummaryResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = MaintenanceIntegrityReportSummaryResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of MaintenanceIntegrityReportSummaryResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = MaintenanceIntegrityReportSummaryResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'checksum_mismatch', + 'missing_file', + 'orphan_file', + }; +} + diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 8e39950f20..4be3cfadfb 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -323,51 +323,6 @@ } }, "/admin/maintenance": { - "get": { - "description": "...", - "operationId": "getIntegrityReport", - "parameters": [], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/MaintenanceIntegrityReportResponseDto" - } - } - }, - "description": "" - } - }, - "security": [ - { - "bearer": [] - }, - { - "cookie": [] - }, - { - "api_key": [] - } - ], - "summary": "Get integrity report", - "tags": [ - "Maintenance (admin)" - ], - "x-immich-admin-only": true, - "x-immich-history": [ - { - "version": "v9.9.9", - "state": "Added" - }, - { - "version": "v9.9.9", - "state": "Alpha" - } - ], - "x-immich-permission": "maintenance", - "x-immich-state": "Alpha" - }, "post": { "description": "Put Immich into or take it out of maintenance mode", "operationId": "setMaintenanceMode", @@ -417,6 +372,110 @@ "x-immich-state": "Alpha" } }, + "/admin/maintenance/integrity/report": { + "post": { + "description": "...", + "operationId": "getIntegrityReport", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MaintenanceGetIntegrityReportDto" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MaintenanceIntegrityReportResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get integrity report by type", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v9.9.9", + "state": "Added" + }, + { + "version": "v9.9.9", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/maintenance/integrity/summary": { + "get": { + "description": "...", + "operationId": "getIntegrityReportSummary", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MaintenanceIntegrityReportSummaryResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get integrity report summary", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v9.9.9", + "state": "Added" + }, + { + "version": "v9.9.9", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, "/admin/maintenance/login": { "post": { "description": "Login with maintenance token or cookie to receive current information and perform further actions.", @@ -16634,6 +16693,14 @@ ], "type": "string" }, + "IntegrityReportType": { + "enum": [ + "orphan_file", + "missing_file", + "checksum_mismatch" + ], + "type": "string" + }, "JobCreateDto": { "properties": { "name": { @@ -16965,6 +17032,21 @@ ], "type": "object" }, + "MaintenanceGetIntegrityReportDto": { + "properties": { + "type": { + "allOf": [ + { + "$ref": "#/components/schemas/IntegrityReportType" + } + ] + } + }, + "required": [ + "type" + ], + "type": "object" + }, "MaintenanceIntegrityReportDto": { "properties": { "id": { @@ -16974,12 +17056,11 @@ "type": "string" }, "type": { - "enum": [ - "orphan_file", - "missing_file", - "checksum_mismatch" - ], - "type": "string" + "allOf": [ + { + "$ref": "#/components/schemas/IntegrityReportType" + } + ] } }, "required": [ @@ -17003,6 +17084,25 @@ ], "type": "object" }, + "MaintenanceIntegrityReportSummaryResponseDto": { + "properties": { + "checksum_mismatch": { + "type": "integer" + }, + "missing_file": { + "type": "integer" + }, + "orphan_file": { + "type": "integer" + } + }, + "required": [ + "checksum_mismatch", + "missing_file", + "orphan_file" + ], + "type": "object" + }, "MaintenanceLoginDto": { "properties": { "token": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index d4802f2abc..79ed06a7f8 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -40,16 +40,24 @@ export type ActivityStatisticsResponseDto = { comments: number; likes: number; }; +export type SetMaintenanceModeDto = { + action: MaintenanceAction; +}; +export type MaintenanceGetIntegrityReportDto = { + "type": IntegrityReportType; +}; export type MaintenanceIntegrityReportDto = { id: string; path: string; - "type": Type; + "type": IntegrityReportType; }; export type MaintenanceIntegrityReportResponseDto = { items: MaintenanceIntegrityReportDto[]; }; -export type SetMaintenanceModeDto = { - action: MaintenanceAction; +export type MaintenanceIntegrityReportSummaryResponseDto = { + checksum_mismatch: number; + missing_file: number; + orphan_file: number; }; export type MaintenanceLoginDto = { token?: string; @@ -1874,17 +1882,6 @@ export function unlinkAllOAuthAccountsAdmin(opts?: Oazapfts.RequestOpts) { method: "POST" })); } -/** - * Get integrity report - */ -export function getIntegrityReport(opts?: Oazapfts.RequestOpts) { - return oazapfts.ok(oazapfts.fetchJson<{ - status: 200; - data: MaintenanceIntegrityReportResponseDto; - }>("/admin/maintenance", { - ...opts - })); -} /** * Set maintenance mode */ @@ -1897,6 +1894,32 @@ export function setMaintenanceMode({ setMaintenanceModeDto }: { body: setMaintenanceModeDto }))); } +/** + * Get integrity report by type + */ +export function getIntegrityReport({ maintenanceGetIntegrityReportDto }: { + maintenanceGetIntegrityReportDto: MaintenanceGetIntegrityReportDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 201; + data: MaintenanceIntegrityReportResponseDto; + }>("/admin/maintenance/integrity/report", oazapfts.json({ + ...opts, + method: "POST", + body: maintenanceGetIntegrityReportDto + }))); +} +/** + * Get integrity report summary + */ +export function getIntegrityReportSummary(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: MaintenanceIntegrityReportSummaryResponseDto; + }>("/admin/maintenance/integrity/summary", { + ...opts + })); +} /** * Log into maintenance mode */ @@ -5173,15 +5196,15 @@ export enum UserAvatarColor { Gray = "gray", Amber = "amber" } -export enum Type { - OrphanFile = "orphan_file", - MissingFile = "missing_file", - ChecksumMismatch = "checksum_mismatch" -} export enum MaintenanceAction { Start = "start", End = "end" } +export enum IntegrityReportType { + OrphanFile = "orphan_file", + MissingFile = "missing_file", + ChecksumMismatch = "checksum_mismatch" +} export enum NotificationLevel { Success = "success", Error = "error", diff --git a/server/src/controllers/maintenance.controller.ts b/server/src/controllers/maintenance.controller.ts index 36e8003521..8f0d7ab723 100644 --- a/server/src/controllers/maintenance.controller.ts +++ b/server/src/controllers/maintenance.controller.ts @@ -7,6 +7,7 @@ import { MaintenanceAuthDto, MaintenanceGetIntegrityReportDto, MaintenanceIntegrityReportResponseDto, + MaintenanceIntegrityReportSummaryResponseDto, MaintenanceLoginDto, SetMaintenanceModeDto, } from 'src/dtos/maintenance.dto'; @@ -53,14 +54,25 @@ export class MaintenanceController { } } - @Get() + @Get('integrity/summary') @Endpoint({ - summary: 'Get integrity report', + summary: 'Get integrity report summary', description: '...', history: new HistoryBuilder().added('v9.9.9').alpha('v9.9.9'), }) @Authenticated({ permission: Permission.Maintenance, admin: true }) - getIntegrityReport(dto: MaintenanceGetIntegrityReportDto): Promise { + getIntegrityReportSummary(): Promise { + return this.service.getIntegrityReportSummary(); // + } + + @Post('integrity/report') + @Endpoint({ + summary: 'Get integrity report by type', + description: '...', + history: new HistoryBuilder().added('v9.9.9').alpha('v9.9.9'), + }) + @Authenticated({ permission: Permission.Maintenance, admin: true }) + getIntegrityReport(@Body() dto: MaintenanceGetIntegrityReportDto): Promise { return this.service.getIntegrityReport(dto); } } diff --git a/server/src/dtos/maintenance.dto.ts b/server/src/dtos/maintenance.dto.ts index 444b843e92..c4bf26a315 100644 --- a/server/src/dtos/maintenance.dto.ts +++ b/server/src/dtos/maintenance.dto.ts @@ -1,4 +1,4 @@ -import { IsEnum } from 'class-validator'; +import { ApiProperty } from '@nestjs/swagger'; import { IntegrityReportType, MaintenanceAction } from 'src/enum'; import { ValidateEnum, ValidateString } from 'src/validation'; @@ -16,7 +16,19 @@ export class MaintenanceAuthDto { username!: string; } +export class MaintenanceIntegrityReportSummaryResponseDto { + @ApiProperty({ type: 'integer' }) + [IntegrityReportType.ChecksumFail]!: number; + @ApiProperty({ type: 'integer' }) + [IntegrityReportType.MissingFile]!: number; + @ApiProperty({ type: 'integer' }) + [IntegrityReportType.OrphanFile]!: number; +} + export class MaintenanceGetIntegrityReportDto { + @ValidateEnum({ enum: IntegrityReportType, name: 'IntegrityReportType' }) + type!: IntegrityReportType; + // todo: paginate // @IsInt() // @Min(1) @@ -27,7 +39,7 @@ export class MaintenanceGetIntegrityReportDto { class MaintenanceIntegrityReportDto { id!: string; - @IsEnum(IntegrityReportType) + @ValidateEnum({ enum: IntegrityReportType, name: 'IntegrityReportType' }) type!: IntegrityReportType; path!: string; } diff --git a/server/src/repositories/integrity-report.repository.ts b/server/src/repositories/integrity-report.repository.ts index e3b9af6457..9dcf8ca8e6 100644 --- a/server/src/repositories/integrity-report.repository.ts +++ b/server/src/repositories/integrity-report.repository.ts @@ -1,7 +1,12 @@ import { Injectable } from '@nestjs/common'; import { Insertable, Kysely } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; -import { MaintenanceGetIntegrityReportDto, MaintenanceIntegrityReportResponseDto } from 'src/dtos/maintenance.dto'; +import { + MaintenanceGetIntegrityReportDto, + MaintenanceIntegrityReportResponseDto, + MaintenanceIntegrityReportSummaryResponseDto, +} from 'src/dtos/maintenance.dto'; +import { IntegrityReportType } from 'src/enum'; import { DB } from 'src/schema'; import { IntegrityReportTable } from 'src/schema/tables/integrity-report.table'; @@ -18,11 +23,36 @@ export class IntegrityReportRepository { .executeTakeFirst(); } - async getIntegrityReport(_dto: MaintenanceGetIntegrityReportDto): Promise { + async getIntegrityReportSummary(): Promise { + return await this.db + .selectFrom('integrity_report') + .select((eb) => + eb.fn + .countAll() + .filterWhere('type', '=', IntegrityReportType.ChecksumFail) + .as(IntegrityReportType.ChecksumFail), + ) + .select((eb) => + eb.fn + .countAll() + .filterWhere('type', '=', IntegrityReportType.MissingFile) + .as(IntegrityReportType.MissingFile), + ) + .select((eb) => + eb.fn + .countAll() + .filterWhere('type', '=', IntegrityReportType.OrphanFile) + .as(IntegrityReportType.OrphanFile), + ) + .executeTakeFirstOrThrow(); + } + + async getIntegrityReport(dto: MaintenanceGetIntegrityReportDto): Promise { return { items: await this.db .selectFrom('integrity_report') .select(['id', 'type', 'path']) + .where('type', '=', dto.type) .orderBy('createdAt', 'desc') .execute(), }; diff --git a/server/src/services/maintenance.service.ts b/server/src/services/maintenance.service.ts index 7438acfc1d..af41eeed8f 100644 --- a/server/src/services/maintenance.service.ts +++ b/server/src/services/maintenance.service.ts @@ -4,6 +4,7 @@ import { MaintenanceAuthDto, MaintenanceGetIntegrityReportDto, MaintenanceIntegrityReportResponseDto, + MaintenanceIntegrityReportSummaryResponseDto, } from 'src/dtos/maintenance.dto'; import { SystemMetadataKey } from 'src/enum'; import { BaseService } from 'src/services/base.service'; @@ -55,6 +56,10 @@ export class MaintenanceService extends BaseService { return await createMaintenanceLoginUrl(baseUrl, auth, secret); } + getIntegrityReportSummary(): Promise { + return this.integrityReportRepository.getIntegrityReportSummary(); + } + getIntegrityReport(dto: MaintenanceGetIntegrityReportDto): Promise { return this.integrityReportRepository.getIntegrityReport(dto); } diff --git a/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte b/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte index 61d4c643c8..f48d48805d 100644 --- a/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte +++ b/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte @@ -1,15 +1,17 @@ @@ -11,7 +11,7 @@ - + diff --git a/web/src/routes/admin/maintenance/+page.svelte b/web/src/routes/admin/maintenance/+page.svelte index 12a0e4067d..b16fec94be 100644 --- a/web/src/routes/admin/maintenance/+page.svelte +++ b/web/src/routes/admin/maintenance/+page.svelte @@ -1,12 +1,11 @@ + + + + +
+
+ + + + + + + + + {#each data.integrityReport.items as { id, path } (id)} + + + + + {/each} + +
{$t('filename')}
{path} +
+
+
+ diff --git a/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts new file mode 100644 index 0000000000..9c6b73ec0e --- /dev/null +++ b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts @@ -0,0 +1,23 @@ +import { authenticate } from '$lib/utils/auth'; +import { getFormatter } from '$lib/utils/i18n'; +import { getIntegrityReport, IntegrityReportType } from '@immich/sdk'; +import type { PageLoad } from './$types'; + +export const load = (async ({ params, url }) => { + const type = params.type as IntegrityReportType; + + await authenticate(url, { admin: true }); + const integrityReport = await getIntegrityReport({ + maintenanceGetIntegrityReportDto: { + type, + }, + }); + const $t = await getFormatter(); + + return { + integrityReport, + meta: { + title: $t(`admin.maintenance_integrity_${type}`), + }, + }; +}) satisfies PageLoad; From c50118e5358239bfabda289b4f590d1a09183779 Mon Sep 17 00:00:00 2001 From: izzy Date: Fri, 28 Nov 2025 12:10:41 +0000 Subject: [PATCH 19/50] chore: remove old table comment --- web/src/routes/admin/maintenance/+page.svelte | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/web/src/routes/admin/maintenance/+page.svelte b/web/src/routes/admin/maintenance/+page.svelte index b16fec94be..1f65dee3d0 100644 --- a/web/src/routes/admin/maintenance/+page.svelte +++ b/web/src/routes/admin/maintenance/+page.svelte @@ -63,48 +63,6 @@ {/each} - - From 13e9cf0ed9841e83fa86c1465cba92fb28d4f966 Mon Sep 17 00:00:00 2001 From: izzy Date: Fri, 28 Nov 2025 12:50:30 +0000 Subject: [PATCH 20/50] stash: moving computers because pnpm is cooked --- e2e/src/api/specs/maintenance.e2e-spec.ts | 21 ++++++++++++++++--- .../src/controllers/maintenance.controller.ts | 2 +- .../src/services/maintenance.service.spec.ts | 3 +++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/e2e/src/api/specs/maintenance.e2e-spec.ts b/e2e/src/api/specs/maintenance.e2e-spec.ts index b6c7540bc5..73f46e5ee7 100644 --- a/e2e/src/api/specs/maintenance.e2e-spec.ts +++ b/e2e/src/api/specs/maintenance.e2e-spec.ts @@ -34,9 +34,24 @@ describe('/admin/maintenance', () => { }); }); + describe('POST /integrity/summary', async () => { + it('should report no issues', async () => { + const { status, body } = await request(app) + .get('/admin/maintenance/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + expect(status).toBe(200); + expect(body).toEqual({ + missing_file: 0, + orphan_file: 0, + checksum_mismatch: 0, + }); + }); + }); + // => enter maintenance mode - describe.sequential('POST /', () => { + describe.skip.sequential('POST /', () => { it('should require authentication', async () => { const { status, body } = await request(app).post('/admin/maintenance').send({ action: 'end', @@ -93,7 +108,7 @@ describe('/admin/maintenance', () => { // => in maintenance mode - describe.sequential('in maintenance mode', () => { + describe.skip.sequential('in maintenance mode', () => { describe('GET ~/server/config', async () => { it('should indicate we are in maintenance mode', async () => { const { status, body } = await request(app).get('/server/config'); @@ -147,7 +162,7 @@ describe('/admin/maintenance', () => { // => exit maintenance mode - describe.sequential('POST /', () => { + describe.skip.sequential('POST /', () => { it('should exit maintenance mode', async () => { const { status } = await request(app).post('/admin/maintenance').set('cookie', cookie!).send({ action: 'end', diff --git a/server/src/controllers/maintenance.controller.ts b/server/src/controllers/maintenance.controller.ts index 8f0d7ab723..b41e892c61 100644 --- a/server/src/controllers/maintenance.controller.ts +++ b/server/src/controllers/maintenance.controller.ts @@ -62,7 +62,7 @@ export class MaintenanceController { }) @Authenticated({ permission: Permission.Maintenance, admin: true }) getIntegrityReportSummary(): Promise { - return this.service.getIntegrityReportSummary(); // + return this.service.getIntegrityReportSummary(); } @Post('integrity/report') diff --git a/server/src/services/maintenance.service.spec.ts b/server/src/services/maintenance.service.spec.ts index cc497a6ea4..5996d6b3c1 100644 --- a/server/src/services/maintenance.service.spec.ts +++ b/server/src/services/maintenance.service.spec.ts @@ -106,4 +106,7 @@ describe(MaintenanceService.name, () => { expect(mocks.systemMetadata.get).toHaveBeenCalledTimes(1); }); }); + + describe.skip('getIntegrityReportSummary'); // just calls repository + describe.skip('getIntegrityReport'); // just calls repository }); From 2779fce7d062cfb06818f2e1a05afd12494a2891 Mon Sep 17 00:00:00 2001 From: izzy Date: Fri, 28 Nov 2025 15:27:12 +0000 Subject: [PATCH 21/50] feat: manually trigger integrity jobs feat: update summary after job runs --- i18n/en.json | 6 + mobile/openapi/lib/model/manual_job_name.dart | 18 +++ open-api/immich-openapi-specs.json | 8 +- open-api/typescript-sdk/src/fetch-client.ts | 8 +- server/src/enum.ts | 18 ++- server/src/services/integrity.service.ts | 31 ++++-- server/src/services/job.service.ts | 24 ++++ server/src/types.ts | 10 +- web/src/lib/modals/JobCreateModal.svelte | 24 ++++ web/src/routes/admin/maintenance/+page.svelte | 104 ++++++++++++++++-- 10 files changed, 224 insertions(+), 27 deletions(-) diff --git a/i18n/en.json b/i18n/en.json index 8870b31a9a..838271eae6 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -183,8 +183,14 @@ "maintenance_start_error": "Failed to start maintenance mode.", "maintenance_integrity_report": "Integrity Report", "maintenance_integrity_orphan_file": "Orphan Files", + "maintenance_integrity_orphan_file_job": "Check for orphaned files", + "maintenance_integrity_orphan_file_refresh_job": "Refresh orphan file reports", "maintenance_integrity_missing_file": "Missing Files", + "maintenance_integrity_missing_file_job": "Check for missing files", + "maintenance_integrity_missing_file_refresh_job": "Refresh missing file reports", "maintenance_integrity_checksum_mismatch": "Checksum Mismatch", + "maintenance_integrity_checksum_mismatch_job": "Check for checksum mismatches", + "maintenance_integrity_checksum_mismatch_refresh_job": "Refresh checksum mismatch reports", "manage_concurrency": "Manage Concurrency", "manage_log_settings": "Manage log settings", "map_dark_style": "Dark style", diff --git a/mobile/openapi/lib/model/manual_job_name.dart b/mobile/openapi/lib/model/manual_job_name.dart index 311215ad9e..424dc60e42 100644 --- a/mobile/openapi/lib/model/manual_job_name.dart +++ b/mobile/openapi/lib/model/manual_job_name.dart @@ -29,6 +29,12 @@ class ManualJobName { static const memoryCleanup = ManualJobName._(r'memory-cleanup'); static const memoryCreate = ManualJobName._(r'memory-create'); static const backupDatabase = ManualJobName._(r'backup-database'); + static const integrityMissingFiles = ManualJobName._(r'integrity-missing-files'); + static const integrityOrphanFiles = ManualJobName._(r'integrity-orphan-files'); + static const integrityChecksumMismatch = ManualJobName._(r'integrity-checksum-mismatch'); + static const integrityMissingFilesRefresh = ManualJobName._(r'integrity-missing-files-refresh'); + static const integrityOrphanFilesRefresh = ManualJobName._(r'integrity-orphan-files-refresh'); + static const integrityChecksumMismatchRefresh = ManualJobName._(r'integrity-checksum-mismatch-refresh'); /// List of all possible values in this [enum][ManualJobName]. static const values = [ @@ -38,6 +44,12 @@ class ManualJobName { memoryCleanup, memoryCreate, backupDatabase, + integrityMissingFiles, + integrityOrphanFiles, + integrityChecksumMismatch, + integrityMissingFilesRefresh, + integrityOrphanFilesRefresh, + integrityChecksumMismatchRefresh, ]; static ManualJobName? fromJson(dynamic value) => ManualJobNameTypeTransformer().decode(value); @@ -82,6 +94,12 @@ class ManualJobNameTypeTransformer { case r'memory-cleanup': return ManualJobName.memoryCleanup; case r'memory-create': return ManualJobName.memoryCreate; case r'backup-database': return ManualJobName.backupDatabase; + case r'integrity-missing-files': return ManualJobName.integrityMissingFiles; + case r'integrity-orphan-files': return ManualJobName.integrityOrphanFiles; + case r'integrity-checksum-mismatch': return ManualJobName.integrityChecksumMismatch; + case r'integrity-missing-files-refresh': return ManualJobName.integrityMissingFilesRefresh; + case r'integrity-orphan-files-refresh': return ManualJobName.integrityOrphanFilesRefresh; + case r'integrity-checksum-mismatch-refresh': return ManualJobName.integrityChecksumMismatchRefresh; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 4be3cfadfb..83571c8d71 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -17118,7 +17118,13 @@ "user-cleanup", "memory-cleanup", "memory-create", - "backup-database" + "backup-database", + "integrity-missing-files", + "integrity-orphan-files", + "integrity-checksum-mismatch", + "integrity-missing-files-refresh", + "integrity-orphan-files-refresh", + "integrity-checksum-mismatch-refresh" ], "type": "string" }, diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 79ed06a7f8..0d69c2454f 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -5441,7 +5441,13 @@ export enum ManualJobName { UserCleanup = "user-cleanup", MemoryCleanup = "memory-cleanup", MemoryCreate = "memory-create", - BackupDatabase = "backup-database" + BackupDatabase = "backup-database", + IntegrityMissingFiles = "integrity-missing-files", + IntegrityOrphanFiles = "integrity-orphan-files", + IntegrityChecksumMismatch = "integrity-checksum-mismatch", + IntegrityMissingFilesRefresh = "integrity-missing-files-refresh", + IntegrityOrphanFilesRefresh = "integrity-orphan-files-refresh", + IntegrityChecksumMismatchRefresh = "integrity-checksum-mismatch-refresh" } export enum QueueName { ThumbnailGeneration = "thumbnailGeneration", diff --git a/server/src/enum.ts b/server/src/enum.ts index 0ee2765741..2b51b294ff 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -345,6 +345,12 @@ export enum SourceType { Manual = 'manual', } +export enum IntegrityReportType { + OrphanFile = 'orphan_file', + MissingFile = 'missing_file', + ChecksumFail = 'checksum_mismatch', +} + export enum ManualJobName { PersonCleanup = 'person-cleanup', TagCleanup = 'tag-cleanup', @@ -352,6 +358,12 @@ export enum ManualJobName { MemoryCleanup = 'memory-cleanup', MemoryCreate = 'memory-create', BackupDatabase = 'backup-database', + IntegrityMissingFiles = `integrity-missing-files`, + IntegrityOrphanFiles = `integrity-orphan-files`, + IntegrityChecksumFiles = `integrity-checksum-mismatch`, + IntegrityMissingFilesRefresh = `integrity-missing-files-refresh`, + IntegrityOrphanFilesRefresh = `integrity-orphan-files-refresh`, + IntegrityChecksumFilesRefresh = `integrity-checksum-mismatch-refresh`, } export enum AssetPathType { @@ -482,12 +494,6 @@ export enum CacheControl { None = 'none', } -export enum IntegrityReportType { - OrphanFile = 'orphan_file', - MissingFile = 'missing_file', - ChecksumFail = 'checksum_mismatch', -} - export enum ImmichEnvironment { Development = 'development', Testing = 'testing', diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index fcc7589642..06639c9395 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -19,7 +19,7 @@ import { } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; -import { IIntegrityOrphanedFilesJob, IIntegrityPathWithReportJob } from 'src/types'; +import { IIntegrityJob, IIntegrityOrphanedFilesJob, IIntegrityPathWithReportJob } from 'src/types'; import { handlePromiseError } from 'src/utils/misc'; async function* chunk(generator: AsyncIterableIterator, n: number) { @@ -130,7 +130,7 @@ export class IntegrityService extends BaseService { } @OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.BackgroundTask }) - async handleOrphanedFilesQueueAll(): Promise { + async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise { this.logger.log(`Checking for out of date orphaned file reports...`); const reports = this.assetJobRepository.streamIntegrityReports(IntegrityReportType.OrphanFile); @@ -148,6 +148,11 @@ export class IntegrityService extends BaseService { this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`); } + if (refreshOnly) { + this.logger.log('Refresh complete.'); + return JobStatus.Success; + } + this.logger.log(`Scanning for orphaned files...`); const assetPaths = this.storageRepository.walk({ @@ -232,8 +237,8 @@ export class IntegrityService extends BaseService { const results = await Promise.all( paths.map(({ reportId, path }) => stat(path) - .then(() => reportId) - .catch(() => void 0), + .then(() => void 0) + .catch(() => reportId), ), ); @@ -243,12 +248,18 @@ export class IntegrityService extends BaseService { await this.integrityReportRepository.deleteByIds(reportIds); } - this.logger.log(`Processed ${paths.length} and found ${reportIds.length} orphaned file(s).`); + this.logger.log(`Processed ${paths.length} paths and found ${reportIds.length} report(s) out of date.`); return JobStatus.Success; } @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.BackgroundTask }) - async handleMissingFilesQueueAll(): Promise { + async handleMissingFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise { + if (refreshOnly) { + // TODO + this.logger.log('Refresh complete.'); + return JobStatus.Success; + } + this.logger.log(`Scanning for missing files...`); const assetPaths = this.assetJobRepository.streamAssetPaths(); @@ -304,7 +315,13 @@ export class IntegrityService extends BaseService { } @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.BackgroundTask }) - async handleChecksumFiles(): Promise { + async handleChecksumFiles({ refreshOnly }: IIntegrityJob = {}): Promise { + if (refreshOnly) { + // TODO + this.logger.log('Refresh complete.'); + return JobStatus.Success; + } + const timeLimit = 60 * 60 * 1000; // 1000; const percentageLimit = 1; // 0.25; diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index b57a203788..cdb7f06e4e 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -34,6 +34,30 @@ const asJobItem = (dto: JobCreateDto): JobItem => { return { name: JobName.DatabaseBackup }; } + case ManualJobName.IntegrityMissingFiles: { + return { name: JobName.IntegrityMissingFilesQueueAll }; + } + + case ManualJobName.IntegrityOrphanFiles: { + return { name: JobName.IntegrityOrphanedFilesQueueAll }; + } + + case ManualJobName.IntegrityChecksumFiles: { + return { name: JobName.IntegrityChecksumFiles }; + } + + case ManualJobName.IntegrityMissingFilesRefresh: { + return { name: JobName.IntegrityMissingFilesQueueAll, data: { refreshOnly: true } }; + } + + case ManualJobName.IntegrityOrphanFilesRefresh: { + return { name: JobName.IntegrityOrphanedFilesQueueAll, data: { refreshOnly: true } }; + } + + case ManualJobName.IntegrityChecksumFilesRefresh: { + return { name: JobName.IntegrityChecksumFiles, data: { refreshOnly: true } }; + } + default: { throw new BadRequestException('Invalid job name'); } diff --git a/server/src/types.ts b/server/src/types.ts index b02626ce26..5110b83836 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -282,6 +282,10 @@ export interface IWorkflowJob { event: WorkflowData[T]; } +export interface IIntegrityJob { + refreshOnly?: boolean; +} + export interface IIntegrityOrphanedFilesJob { type: 'asset' | 'asset_file'; paths: string[]; @@ -403,12 +407,12 @@ export type JobItem = | { name: JobName.WorkflowRun; data: IWorkflowJob } // Integrity - | { name: JobName.IntegrityOrphanedFilesQueueAll; data: IBaseJob } + | { name: JobName.IntegrityOrphanedFilesQueueAll; data?: IIntegrityJob } | { name: JobName.IntegrityOrphanedFiles; data: IIntegrityOrphanedFilesJob } | { name: JobName.IntegrityOrphanedCheckReports; data: IIntegrityPathWithReportJob } - | { name: JobName.IntegrityMissingFilesQueueAll; data: IBaseJob } + | { name: JobName.IntegrityMissingFilesQueueAll; data?: IIntegrityJob } | { name: JobName.IntegrityMissingFiles; data: IIntegrityPathWithReportJob } - | { name: JobName.IntegrityChecksumFiles; data: IBaseJob }; + | { name: JobName.IntegrityChecksumFiles; data?: IIntegrityJob }; export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number]; diff --git a/web/src/lib/modals/JobCreateModal.svelte b/web/src/lib/modals/JobCreateModal.svelte index 1b9ea09032..ec050f87af 100644 --- a/web/src/lib/modals/JobCreateModal.svelte +++ b/web/src/lib/modals/JobCreateModal.svelte @@ -16,6 +16,30 @@ { title: $t('admin.memory_cleanup_job'), value: ManualJobName.MemoryCleanup }, { title: $t('admin.memory_generate_job'), value: ManualJobName.MemoryCreate }, { title: $t('admin.backup_database'), value: ManualJobName.BackupDatabase }, + { + title: $t('admin.maintenance_integrity_missing_file_job'), + value: ManualJobName.IntegrityMissingFiles, + }, + { + title: $t('admin.maintenance_integrity_orphan_file_job'), + value: ManualJobName.IntegrityOrphanFiles, + }, + { + title: $t('admin.maintenance_integrity_checksum_mismatch_job'), + value: ManualJobName.IntegrityChecksumMismatch, + }, + { + title: $t('admin.maintenance_integrity_missing_file_refresh_job'), + value: ManualJobName.IntegrityMissingFilesRefresh, + }, + { + title: $t('admin.maintenance_integrity_orphan_file_refresh_job'), + value: ManualJobName.IntegrityOrphanFilesRefresh, + }, + { + title: $t('admin.maintenance_integrity_checksum_mismatch_refresh_job'), + value: ManualJobName.IntegrityChecksumMismatchRefresh, + }, ].map(({ value, title }) => ({ id: value, label: title, value })); let selectedJob: ComboBoxOption | undefined = $state(undefined); diff --git a/web/src/routes/admin/maintenance/+page.svelte b/web/src/routes/admin/maintenance/+page.svelte index 1f65dee3d0..61b97ea6f9 100644 --- a/web/src/routes/admin/maintenance/+page.svelte +++ b/web/src/routes/admin/maintenance/+page.svelte @@ -2,10 +2,22 @@ import AdminPageLayout from '$lib/components/layouts/AdminPageLayout.svelte'; import ServerStatisticsCard from '$lib/components/server-statistics/ServerStatisticsCard.svelte'; import { AppRoute } from '$lib/constants'; + import { asyncTimeout } from '$lib/utils'; import { handleError } from '$lib/utils/handle-error'; - import { MaintenanceAction, setMaintenanceMode } from '@immich/sdk'; - import { Button, HStack, Text } from '@immich/ui'; + import { + createJob, + getIntegrityReportSummary, + getQueuesLegacy, + IntegrityReportType, + MaintenanceAction, + ManualJobName, + setMaintenanceMode, + type MaintenanceIntegrityReportSummaryResponseDto, + type QueuesResponseLegacyDto, + } from '@immich/sdk'; + import { Button, HStack, Text, toastManager } from '@immich/ui'; import { mdiProgressWrench } from '@mdi/js'; + import { onDestroy, onMount } from 'svelte'; import { t } from 'svelte-i18n'; import type { PageData } from './$types'; @@ -15,6 +27,14 @@ let { data }: Props = $props(); + let integrityReport: MaintenanceIntegrityReportSummaryResponseDto | undefined = $state(data.integrityReport); + + const TYPES: IntegrityReportType[] = [ + IntegrityReportType.OrphanFile, + IntegrityReportType.MissingFile, + IntegrityReportType.ChecksumMismatch, + ]; + async function switchToMaintenance() { try { await setMaintenanceMode({ @@ -26,6 +46,56 @@ handleError(error, $t('admin.maintenance_start_error')); } } + + let jobs: QueuesResponseLegacyDto | undefined = $state(); + let expectingUpdate: boolean = $state(false); + + async function runJob(reportType: IntegrityReportType, refreshOnly?: boolean) { + let name: ManualJobName; + switch (reportType) { + case IntegrityReportType.OrphanFile: { + name = refreshOnly ? ManualJobName.IntegrityOrphanFilesRefresh : ManualJobName.IntegrityOrphanFiles; + break; + } + case IntegrityReportType.MissingFile: { + name = refreshOnly ? ManualJobName.IntegrityMissingFilesRefresh : ManualJobName.IntegrityMissingFiles; + break; + } + case IntegrityReportType.ChecksumMismatch: { + name = refreshOnly ? ManualJobName.IntegrityChecksumMismatchRefresh : ManualJobName.IntegrityChecksumMismatch; + break; + } + } + + try { + await createJob({ jobCreateDto: { name } }); + if (jobs) { + expectingUpdate = true; + jobs.backgroundTask.queueStatus.isActive = true; + } + toastManager.success($t('admin.job_created')); + } catch (error) { + handleError(error, $t('errors.unable_to_submit_job')); + } + } + + let running = true; + + onMount(async () => { + while (running) { + jobs = await getQueuesLegacy(); + if (jobs.backgroundTask.queueStatus.isActive) { + expectingUpdate = true; + } else if (expectingUpdate) { + integrityReport = await getIntegrityReportSummary(); + } + await asyncTimeout(5000); + } + }); + + onDestroy(() => { + running = false; + }); @@ -48,17 +118,33 @@

{$t('admin.maintenance_integrity_report')}