diff --git a/e2e/src/api/specs/integrity.e2e-spec.ts b/e2e/src/api/specs/integrity.e2e-spec.ts new file mode 100644 index 0000000000..17f8e22a3a --- /dev/null +++ b/e2e/src/api/specs/integrity.e2e-spec.ts @@ -0,0 +1,198 @@ +import { IntegrityReportType, LoginResponseDto, ManualJobName, QueueName } from '@immich/sdk'; +import { readFile } from 'node:fs/promises'; +import { app, testAssetDir, utils } from 'src/utils'; +import request from 'supertest'; +import { afterEach, beforeAll, describe, expect, it } from 'vitest'; + +const assetFilepath = `${testAssetDir}/metadata/gps-position/thompson-springs.jpg`; + +describe('/admin/integrity', () => { + let admin: LoginResponseDto; + + beforeAll(async () => { + await utils.resetDatabase(); + admin = await utils.adminSetup(); + }); + + describe('POST /summary (& jobs)', async () => { + let baseline: Record; + + beforeAll(async () => { + await utils.createAsset(admin.accessToken, { + assetData: { + filename: 'asset.jpg', + bytes: await readFile(assetFilepath), + }, + }); + + await utils.copyFolder(`/data/upload/${admin.userId}`, `/data/upload/${admin.userId}-bak`); + }); + + afterEach(async () => { + await utils.deleteFolder(`/data/upload/${admin.userId}`); + await utils.copyFolder(`/data/upload/${admin.userId}-bak`, `/data/upload/${admin.userId}`); + }); + + it.sequential('may report issues', async () => { + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityOrphanFiles, + }); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFiles, + }); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatch, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual({ + missing_file: 0, + orphan_file: expect.any(Number), + checksum_mismatch: 0, + }); + + baseline = body; + }); + + it.sequential('should detect an orphan file (job: check orphan files)', async () => { + await utils.putTextFile('orphan', `/data/upload/${admin.userId}/orphan1.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityOrphanFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + orphan_file: baseline.orphan_file + 1, + }), + ); + }); + + it.sequential('should detect outdated orphan file reports (job: refresh orphan files)', async () => { + // these should not be detected: + await utils.putTextFile('orphan', `/data/upload/${admin.userId}/orphan2.png`); + await utils.putTextFile('orphan', `/data/upload/${admin.userId}/orphan3.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityOrphanFilesRefresh, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + orphan_file: baseline.orphan_file, + }), + ); + }); + + it.sequential('should detect a missing file and not a checksum mismatch (job: check missing files)', async () => { + await utils.deleteFolder(`/data/upload/${admin.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + missing_file: 1, + checksum_mismatch: 0, + }), + ); + }); + + it.sequential('should detect outdated missing file reports (job: refresh missing files)', async () => { + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFilesRefresh, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + missing_file: 0, + checksum_mismatch: 0, + }), + ); + }); + + it.sequential('should detect a checksum mismatch (job: check file checksums)', async () => { + await utils.truncateFolder(`/data/upload/${admin.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatch, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + checksum_mismatch: 1, + }), + ); + }); + + it.sequential('should detect outdated checksum mismatch reports (job: refresh file checksums)', async () => { + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatchRefresh, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + checksum_mismatch: 0, + }), + ); + }); + }); +}); diff --git a/e2e/src/api/specs/maintenance.e2e-spec.ts b/e2e/src/api/specs/maintenance.e2e-spec.ts index b6c7540bc5..9e010e60d4 100644 --- a/e2e/src/api/specs/maintenance.e2e-spec.ts +++ b/e2e/src/api/specs/maintenance.e2e-spec.ts @@ -83,8 +83,8 @@ describe('/admin/maintenance', () => { return body.maintenanceMode; }, { - interval: 5e2, - timeout: 1e4, + interval: 500, + timeout: 60_000, }, ) .toBeTruthy(); @@ -162,8 +162,8 @@ describe('/admin/maintenance', () => { return body.maintenanceMode; }, { - interval: 5e2, - timeout: 1e4, + interval: 500, + timeout: 60_000, }, ) .toBeFalsy(); diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts index 15bb112cd8..204e509446 100644 --- a/e2e/src/utils.ts +++ b/e2e/src/utils.ts @@ -6,6 +6,7 @@ import { CheckExistingAssetsDto, CreateAlbumDto, CreateLibraryDto, + JobCreateDto, MaintenanceAction, MetadataSearchDto, Permission, @@ -21,6 +22,7 @@ import { checkExistingAssets, createAlbum, createApiKey, + createJob, createLibrary, createPartner, createPerson, @@ -52,9 +54,12 @@ import { import { BrowserContext } from '@playwright/test'; import { exec, spawn } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { createWriteStream, existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtemp } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { dirname, resolve } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; import { setTimeout as setAsyncTimeout } from 'node:timers/promises'; import { promisify } from 'node:util'; import pg from 'pg'; @@ -171,6 +176,7 @@ export const utils = { 'user', 'system_metadata', 'tag', + 'integrity_report', ]; const sql: string[] = []; @@ -481,6 +487,9 @@ export const utils = { tagAssets: (accessToken: string, tagId: string, assetIds: string[]) => tagAssets({ id: tagId, bulkIdsDto: { ids: assetIds } }, { headers: asBearerAuth(accessToken) }), + createJob: async (accessToken: string, jobCreateDto: JobCreateDto) => + createJob({ jobCreateDto }, { headers: asBearerAuth(accessToken) }), + queueCommand: async (accessToken: string, name: QueueName, queueCommandDto: QueueCommandDto) => runQueueCommandLegacy({ name, queueCommandDto }, { headers: asBearerAuth(accessToken) }), @@ -559,6 +568,50 @@ export const utils = { mkdirSync(`${testAssetDir}/temp`, { recursive: true }); }, + putFile(source: string, dest: string) { + return executeCommand('docker', ['cp', source, `immich-e2e-server:${dest}`]).promise; + }, + + async putTextFile(contents: string, dest: string) { + const dir = await mkdtemp(join(tmpdir(), 'test-')); + const fn = join(dir, 'file'); + await pipeline(Readable.from(contents), createWriteStream(fn)); + return executeCommand('docker', ['cp', fn, `immich-e2e-server:${dest}`]).promise; + }, + + async move(source: string, dest: string) { + return executeCommand('docker', ['exec', 'immich-e2e-server', 'mv', source, dest]).promise; + }, + + async copyFolder(source: string, dest: string) { + return executeCommand('docker', ['exec', 'immich-e2e-server', 'cp', '-r', source, dest]).promise; + }, + + async deleteFile(path: string) { + return executeCommand('docker', ['exec', 'immich-e2e-server', 'rm', path]).promise; + }, + + async deleteFolder(path: string) { + return executeCommand('docker', ['exec', 'immich-e2e-server', 'rm', '-r', path]).promise; + }, + + async truncateFolder(path: string) { + return executeCommand('docker', [ + 'exec', + 'immich-e2e-server', + 'find', + path, + '-type', + 'f', + '-exec', + 'truncate', + '-s', + '1', + '{}', + ';', + ]).promise; + }, + resetAdminConfig: async (accessToken: string) => { const defaultConfig = await getConfigDefaults({ headers: asBearerAuth(accessToken) }); await updateConfig({ systemConfigDto: defaultConfig }, { headers: asBearerAuth(accessToken) }); diff --git a/i18n/en.json b/i18n/en.json index 5903d7850e..d5756b4f0f 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -181,6 +181,16 @@ "machine_learning_smart_search_enabled": "Enable smart search", "machine_learning_smart_search_enabled_description": "If disabled, images will not be encoded for smart search.", "machine_learning_url_description": "The URL of the machine learning server. If more than one URL is provided, each server will be attempted one-at-a-time until one responds successfully, in order from first to last. Servers that don't respond will be temporarily ignored until they come back online.", + "maintenance_integrity_checksum_mismatch": "Checksum Mismatch", + "maintenance_integrity_checksum_mismatch_job": "Check for checksum mismatches", + "maintenance_integrity_checksum_mismatch_refresh_job": "Refresh checksum mismatch reports", + "maintenance_integrity_missing_file": "Missing Files", + "maintenance_integrity_missing_file_job": "Check for missing files", + "maintenance_integrity_missing_file_refresh_job": "Refresh missing file reports", + "maintenance_integrity_orphan_file": "Orphan Files", + "maintenance_integrity_orphan_file_job": "Check for orphaned files", + "maintenance_integrity_orphan_file_refresh_job": "Refresh orphan file reports", + "maintenance_integrity_report": "Integrity Report", "maintenance_settings": "Maintenance", "maintenance_settings_description": "Put Immich into maintenance mode.", "maintenance_start": "Start maintenance mode", diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 1b7b0e8c82..2166391e39 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -161,6 +161,11 @@ Class | Method | HTTP request | Description *LibrariesApi* | [**scanLibrary**](doc//LibrariesApi.md#scanlibrary) | **POST** /libraries/{id}/scan | Scan a library *LibrariesApi* | [**updateLibrary**](doc//LibrariesApi.md#updatelibrary) | **PUT** /libraries/{id} | Update a library *LibrariesApi* | [**validate**](doc//LibrariesApi.md#validate) | **POST** /libraries/{id}/validate | Validate library settings +*MaintenanceAdminApi* | [**deleteIntegrityReport**](doc//MaintenanceAdminApi.md#deleteintegrityreport) | **DELETE** /admin/integrity/report/{id} | Delete report entry and perform corresponding deletion action +*MaintenanceAdminApi* | [**getIntegrityReport**](doc//MaintenanceAdminApi.md#getintegrityreport) | **POST** /admin/integrity/report | Get integrity report by type +*MaintenanceAdminApi* | [**getIntegrityReportCsv**](doc//MaintenanceAdminApi.md#getintegrityreportcsv) | **GET** /admin/integrity/report/{type}/csv | Export integrity report by type as CSV +*MaintenanceAdminApi* | [**getIntegrityReportFile**](doc//MaintenanceAdminApi.md#getintegrityreportfile) | **GET** /admin/integrity/report/{id}/file | Download the orphan/broken file if one exists +*MaintenanceAdminApi* | [**getIntegrityReportSummary**](doc//MaintenanceAdminApi.md#getintegrityreportsummary) | **GET** /admin/integrity/summary | Get integrity report summary *MaintenanceAdminApi* | [**maintenanceLogin**](doc//MaintenanceAdminApi.md#maintenancelogin) | **POST** /admin/maintenance/login | Log into maintenance mode *MaintenanceAdminApi* | [**setMaintenanceMode**](doc//MaintenanceAdminApi.md#setmaintenancemode) | **POST** /admin/maintenance | Set maintenance mode *MapApi* | [**getMapMarkers**](doc//MapApi.md#getmapmarkers) | **GET** /map/markers | Retrieve map markers @@ -402,6 +407,11 @@ Class | Method | HTTP request | Description - [FoldersResponse](doc//FoldersResponse.md) - [FoldersUpdate](doc//FoldersUpdate.md) - [ImageFormat](doc//ImageFormat.md) + - [IntegrityGetReportDto](doc//IntegrityGetReportDto.md) + - [IntegrityReportDto](doc//IntegrityReportDto.md) + - [IntegrityReportResponseDto](doc//IntegrityReportResponseDto.md) + - [IntegrityReportSummaryResponseDto](doc//IntegrityReportSummaryResponseDto.md) + - [IntegrityReportType](doc//IntegrityReportType.md) - [JobCreateDto](doc//JobCreateDto.md) - [JobName](doc//JobName.md) - [JobSettingsDto](doc//JobSettingsDto.md) @@ -569,6 +579,9 @@ Class | Method | HTTP request | Description - [SystemConfigGeneratedFullsizeImageDto](doc//SystemConfigGeneratedFullsizeImageDto.md) - [SystemConfigGeneratedImageDto](doc//SystemConfigGeneratedImageDto.md) - [SystemConfigImageDto](doc//SystemConfigImageDto.md) + - [SystemConfigIntegrityChecks](doc//SystemConfigIntegrityChecks.md) + - [SystemConfigIntegrityChecksumJob](doc//SystemConfigIntegrityChecksumJob.md) + - [SystemConfigIntegrityJob](doc//SystemConfigIntegrityJob.md) - [SystemConfigJobDto](doc//SystemConfigJobDto.md) - [SystemConfigLibraryDto](doc//SystemConfigLibraryDto.md) - [SystemConfigLibraryScanDto](doc//SystemConfigLibraryScanDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 21730074aa..1d5831de5b 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -154,6 +154,11 @@ part 'model/facial_recognition_config.dart'; part 'model/folders_response.dart'; part 'model/folders_update.dart'; part 'model/image_format.dart'; +part 'model/integrity_get_report_dto.dart'; +part 'model/integrity_report_dto.dart'; +part 'model/integrity_report_response_dto.dart'; +part 'model/integrity_report_summary_response_dto.dart'; +part 'model/integrity_report_type.dart'; part 'model/job_create_dto.dart'; part 'model/job_name.dart'; part 'model/job_settings_dto.dart'; @@ -321,6 +326,9 @@ part 'model/system_config_faces_dto.dart'; part 'model/system_config_generated_fullsize_image_dto.dart'; part 'model/system_config_generated_image_dto.dart'; part 'model/system_config_image_dto.dart'; +part 'model/system_config_integrity_checks.dart'; +part 'model/system_config_integrity_checksum_job.dart'; +part 'model/system_config_integrity_job.dart'; part 'model/system_config_job_dto.dart'; part 'model/system_config_library_dto.dart'; part 'model/system_config_library_scan_dto.dart'; diff --git a/mobile/openapi/lib/api/maintenance_admin_api.dart b/mobile/openapi/lib/api/maintenance_admin_api.dart index 7e46f96c6e..ed9dbf0a2d 100644 --- a/mobile/openapi/lib/api/maintenance_admin_api.dart +++ b/mobile/openapi/lib/api/maintenance_admin_api.dart @@ -16,6 +16,273 @@ class MaintenanceAdminApi { final ApiClient apiClient; + /// Delete report entry and perform corresponding deletion action + /// + /// ... + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [String] id (required): + Future deleteIntegrityReportWithHttpInfo(String id,) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/report/{id}' + .replaceAll('{id}', id); + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'DELETE', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Delete report entry and perform corresponding deletion action + /// + /// ... + /// + /// Parameters: + /// + /// * [String] id (required): + Future deleteIntegrityReport(String id,) async { + final response = await deleteIntegrityReportWithHttpInfo(id,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + } + + /// Get integrity report by type + /// + /// ... + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [IntegrityGetReportDto] integrityGetReportDto (required): + Future getIntegrityReportWithHttpInfo(IntegrityGetReportDto integrityGetReportDto,) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/report'; + + // ignore: prefer_final_locals + Object? postBody = integrityGetReportDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + apiPath, + 'POST', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Get integrity report by type + /// + /// ... + /// + /// Parameters: + /// + /// * [IntegrityGetReportDto] integrityGetReportDto (required): + Future getIntegrityReport(IntegrityGetReportDto integrityGetReportDto,) async { + final response = await getIntegrityReportWithHttpInfo(integrityGetReportDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'IntegrityReportResponseDto',) as IntegrityReportResponseDto; + + } + return null; + } + + /// Export integrity report by type as CSV + /// + /// ... + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [IntegrityReportType] type (required): + Future getIntegrityReportCsvWithHttpInfo(IntegrityReportType type,) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/report/{type}/csv' + .replaceAll('{type}', type.toString()); + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Export integrity report by type as CSV + /// + /// ... + /// + /// Parameters: + /// + /// * [IntegrityReportType] type (required): + Future getIntegrityReportCsv(IntegrityReportType type,) async { + final response = await getIntegrityReportCsvWithHttpInfo(type,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'MultipartFile',) as MultipartFile; + + } + return null; + } + + /// Download the orphan/broken file if one exists + /// + /// ... + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [String] id (required): + Future getIntegrityReportFileWithHttpInfo(String id,) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/report/{id}/file' + .replaceAll('{id}', id); + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Download the orphan/broken file if one exists + /// + /// ... + /// + /// Parameters: + /// + /// * [String] id (required): + Future getIntegrityReportFile(String id,) async { + final response = await getIntegrityReportFileWithHttpInfo(id,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'MultipartFile',) as MultipartFile; + + } + return null; + } + + /// Get integrity report summary + /// + /// ... + /// + /// Note: This method returns the HTTP [Response]. + Future getIntegrityReportSummaryWithHttpInfo() async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/summary'; + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Get integrity report summary + /// + /// ... + Future getIntegrityReportSummary() async { + final response = await getIntegrityReportSummaryWithHttpInfo(); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'IntegrityReportSummaryResponseDto',) as IntegrityReportSummaryResponseDto; + + } + return null; + } + /// Log into maintenance mode /// /// Login with maintenance token or cookie to receive current information and perform further actions. diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 041be67015..64f1c6b836 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -356,6 +356,16 @@ class ApiClient { return FoldersUpdate.fromJson(value); case 'ImageFormat': return ImageFormatTypeTransformer().decode(value); + case 'IntegrityGetReportDto': + return IntegrityGetReportDto.fromJson(value); + case 'IntegrityReportDto': + return IntegrityReportDto.fromJson(value); + case 'IntegrityReportResponseDto': + return IntegrityReportResponseDto.fromJson(value); + case 'IntegrityReportSummaryResponseDto': + return IntegrityReportSummaryResponseDto.fromJson(value); + case 'IntegrityReportType': + return IntegrityReportTypeTypeTransformer().decode(value); case 'JobCreateDto': return JobCreateDto.fromJson(value); case 'JobName': @@ -690,6 +700,12 @@ class ApiClient { return SystemConfigGeneratedImageDto.fromJson(value); case 'SystemConfigImageDto': return SystemConfigImageDto.fromJson(value); + case 'SystemConfigIntegrityChecks': + return SystemConfigIntegrityChecks.fromJson(value); + case 'SystemConfigIntegrityChecksumJob': + return SystemConfigIntegrityChecksumJob.fromJson(value); + case 'SystemConfigIntegrityJob': + return SystemConfigIntegrityJob.fromJson(value); case 'SystemConfigJobDto': return SystemConfigJobDto.fromJson(value); case 'SystemConfigLibraryDto': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index 2c97eeb314..85477a4f6c 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -94,6 +94,9 @@ String parameterToString(dynamic value) { if (value is ImageFormat) { return ImageFormatTypeTransformer().encode(value).toString(); } + if (value is IntegrityReportType) { + return IntegrityReportTypeTypeTransformer().encode(value).toString(); + } if (value is JobName) { return JobNameTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/integrity_get_report_dto.dart b/mobile/openapi/lib/model/integrity_get_report_dto.dart new file mode 100644 index 0000000000..75fb7bb952 --- /dev/null +++ b/mobile/openapi/lib/model/integrity_get_report_dto.dart @@ -0,0 +1,99 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class IntegrityGetReportDto { + /// Returns a new [IntegrityGetReportDto] instance. + IntegrityGetReportDto({ + required this.type, + }); + + IntegrityReportType type; + + @override + bool operator ==(Object other) => identical(this, other) || other is IntegrityGetReportDto && + other.type == type; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (type.hashCode); + + @override + String toString() => 'IntegrityGetReportDto[type=$type]'; + + Map toJson() { + final json = {}; + json[r'type'] = this.type; + return json; + } + + /// Returns a new [IntegrityGetReportDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static IntegrityGetReportDto? fromJson(dynamic value) { + upgradeDto(value, "IntegrityGetReportDto"); + if (value is Map) { + final json = value.cast(); + + return IntegrityGetReportDto( + type: IntegrityReportType.fromJson(json[r'type'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityGetReportDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = IntegrityGetReportDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of IntegrityGetReportDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = IntegrityGetReportDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'type', + }; +} + diff --git a/mobile/openapi/lib/model/integrity_report_dto.dart b/mobile/openapi/lib/model/integrity_report_dto.dart new file mode 100644 index 0000000000..4b3db429ac --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report_dto.dart @@ -0,0 +1,115 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class IntegrityReportDto { + /// Returns a new [IntegrityReportDto] instance. + IntegrityReportDto({ + required this.id, + required this.path, + required this.type, + }); + + String id; + + String path; + + IntegrityReportType type; + + @override + bool operator ==(Object other) => identical(this, other) || other is IntegrityReportDto && + other.id == id && + other.path == path && + other.type == type; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (id.hashCode) + + (path.hashCode) + + (type.hashCode); + + @override + String toString() => 'IntegrityReportDto[id=$id, path=$path, type=$type]'; + + Map toJson() { + final json = {}; + json[r'id'] = this.id; + json[r'path'] = this.path; + json[r'type'] = this.type; + return json; + } + + /// Returns a new [IntegrityReportDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static IntegrityReportDto? fromJson(dynamic value) { + upgradeDto(value, "IntegrityReportDto"); + if (value is Map) { + final json = value.cast(); + + return IntegrityReportDto( + id: mapValueOfType(json, r'id')!, + path: mapValueOfType(json, r'path')!, + type: IntegrityReportType.fromJson(json[r'type'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReportDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = IntegrityReportDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of IntegrityReportDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = IntegrityReportDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'id', + 'path', + 'type', + }; +} + diff --git a/mobile/openapi/lib/model/integrity_report_response_dto.dart b/mobile/openapi/lib/model/integrity_report_response_dto.dart new file mode 100644 index 0000000000..45213f39e0 --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report_response_dto.dart @@ -0,0 +1,99 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class IntegrityReportResponseDto { + /// Returns a new [IntegrityReportResponseDto] instance. + IntegrityReportResponseDto({ + this.items = const [], + }); + + List items; + + @override + bool operator ==(Object other) => identical(this, other) || other is IntegrityReportResponseDto && + _deepEquality.equals(other.items, items); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (items.hashCode); + + @override + String toString() => 'IntegrityReportResponseDto[items=$items]'; + + Map toJson() { + final json = {}; + json[r'items'] = this.items; + return json; + } + + /// Returns a new [IntegrityReportResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static IntegrityReportResponseDto? fromJson(dynamic value) { + upgradeDto(value, "IntegrityReportResponseDto"); + if (value is Map) { + final json = value.cast(); + + return IntegrityReportResponseDto( + items: IntegrityReportDto.listFromJson(json[r'items']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReportResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = IntegrityReportResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of IntegrityReportResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = IntegrityReportResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'items', + }; +} + diff --git a/mobile/openapi/lib/model/integrity_report_summary_response_dto.dart b/mobile/openapi/lib/model/integrity_report_summary_response_dto.dart new file mode 100644 index 0000000000..4b649b1328 --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report_summary_response_dto.dart @@ -0,0 +1,115 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class IntegrityReportSummaryResponseDto { + /// Returns a new [IntegrityReportSummaryResponseDto] instance. + IntegrityReportSummaryResponseDto({ + required this.checksumMismatch, + required this.missingFile, + required this.orphanFile, + }); + + int checksumMismatch; + + int missingFile; + + int orphanFile; + + @override + bool operator ==(Object other) => identical(this, other) || other is IntegrityReportSummaryResponseDto && + other.checksumMismatch == checksumMismatch && + other.missingFile == missingFile && + other.orphanFile == orphanFile; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (checksumMismatch.hashCode) + + (missingFile.hashCode) + + (orphanFile.hashCode); + + @override + String toString() => 'IntegrityReportSummaryResponseDto[checksumMismatch=$checksumMismatch, missingFile=$missingFile, orphanFile=$orphanFile]'; + + Map toJson() { + final json = {}; + json[r'checksum_mismatch'] = this.checksumMismatch; + json[r'missing_file'] = this.missingFile; + json[r'orphan_file'] = this.orphanFile; + return json; + } + + /// Returns a new [IntegrityReportSummaryResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static IntegrityReportSummaryResponseDto? fromJson(dynamic value) { + upgradeDto(value, "IntegrityReportSummaryResponseDto"); + if (value is Map) { + final json = value.cast(); + + return IntegrityReportSummaryResponseDto( + checksumMismatch: mapValueOfType(json, r'checksum_mismatch')!, + missingFile: mapValueOfType(json, r'missing_file')!, + orphanFile: mapValueOfType(json, r'orphan_file')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReportSummaryResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = IntegrityReportSummaryResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of IntegrityReportSummaryResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = IntegrityReportSummaryResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'checksum_mismatch', + 'missing_file', + 'orphan_file', + }; +} + diff --git a/mobile/openapi/lib/model/integrity_report_type.dart b/mobile/openapi/lib/model/integrity_report_type.dart new file mode 100644 index 0000000000..f027cd6f5a --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report_type.dart @@ -0,0 +1,88 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class IntegrityReportType { + /// Instantiate a new enum with the provided [value]. + const IntegrityReportType._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const orphanFile = IntegrityReportType._(r'orphan_file'); + static const missingFile = IntegrityReportType._(r'missing_file'); + static const checksumMismatch = IntegrityReportType._(r'checksum_mismatch'); + + /// List of all possible values in this [enum][IntegrityReportType]. + static const values = [ + orphanFile, + missingFile, + checksumMismatch, + ]; + + static IntegrityReportType? fromJson(dynamic value) => IntegrityReportTypeTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReportType.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [IntegrityReportType] to String, +/// and [decode] dynamic data back to [IntegrityReportType]. +class IntegrityReportTypeTypeTransformer { + factory IntegrityReportTypeTypeTransformer() => _instance ??= const IntegrityReportTypeTypeTransformer._(); + + const IntegrityReportTypeTypeTransformer._(); + + String encode(IntegrityReportType data) => data.value; + + /// Decodes a [dynamic value][data] to a IntegrityReportType. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + IntegrityReportType? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'orphan_file': return IntegrityReportType.orphanFile; + case r'missing_file': return IntegrityReportType.missingFile; + case r'checksum_mismatch': return IntegrityReportType.checksumMismatch; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [IntegrityReportTypeTypeTransformer] instance. + static IntegrityReportTypeTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/job_name.dart b/mobile/openapi/lib/model/job_name.dart index 038a17a8e6..4572ccf7a4 100644 --- a/mobile/openapi/lib/model/job_name.dart +++ b/mobile/openapi/lib/model/job_name.dart @@ -78,6 +78,15 @@ class JobName { static const ocrQueueAll = JobName._(r'OcrQueueAll'); static const ocr = JobName._(r'Ocr'); static const workflowRun = JobName._(r'WorkflowRun'); + static const integrityOrphanedFilesQueueAll = JobName._(r'IntegrityOrphanedFilesQueueAll'); + static const integrityOrphanedFiles = JobName._(r'IntegrityOrphanedFiles'); + static const integrityOrphanedRefresh = JobName._(r'IntegrityOrphanedRefresh'); + static const integrityMissingFilesQueueAll = JobName._(r'IntegrityMissingFilesQueueAll'); + static const integrityMissingFiles = JobName._(r'IntegrityMissingFiles'); + static const integrityMissingFilesRefresh = JobName._(r'IntegrityMissingFilesRefresh'); + static const integrityChecksumFiles = JobName._(r'IntegrityChecksumFiles'); + static const integrityChecksumFilesRefresh = JobName._(r'IntegrityChecksumFilesRefresh'); + static const integrityReportDelete = JobName._(r'IntegrityReportDelete'); /// List of all possible values in this [enum][JobName]. static const values = [ @@ -136,6 +145,15 @@ class JobName { ocrQueueAll, ocr, workflowRun, + integrityOrphanedFilesQueueAll, + integrityOrphanedFiles, + integrityOrphanedRefresh, + integrityMissingFilesQueueAll, + integrityMissingFiles, + integrityMissingFilesRefresh, + integrityChecksumFiles, + integrityChecksumFilesRefresh, + integrityReportDelete, ]; static JobName? fromJson(dynamic value) => JobNameTypeTransformer().decode(value); @@ -229,6 +247,15 @@ class JobNameTypeTransformer { case r'OcrQueueAll': return JobName.ocrQueueAll; case r'Ocr': return JobName.ocr; case r'WorkflowRun': return JobName.workflowRun; + case r'IntegrityOrphanedFilesQueueAll': return JobName.integrityOrphanedFilesQueueAll; + case r'IntegrityOrphanedFiles': return JobName.integrityOrphanedFiles; + case r'IntegrityOrphanedRefresh': return JobName.integrityOrphanedRefresh; + case r'IntegrityMissingFilesQueueAll': return JobName.integrityMissingFilesQueueAll; + case r'IntegrityMissingFiles': return JobName.integrityMissingFiles; + case r'IntegrityMissingFilesRefresh': return JobName.integrityMissingFilesRefresh; + case r'IntegrityChecksumFiles': return JobName.integrityChecksumFiles; + case r'IntegrityChecksumFilesRefresh': return JobName.integrityChecksumFilesRefresh; + case r'IntegrityReportDelete': return JobName.integrityReportDelete; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/mobile/openapi/lib/model/manual_job_name.dart b/mobile/openapi/lib/model/manual_job_name.dart index 311215ad9e..60d14e6ef7 100644 --- a/mobile/openapi/lib/model/manual_job_name.dart +++ b/mobile/openapi/lib/model/manual_job_name.dart @@ -29,6 +29,15 @@ class ManualJobName { static const memoryCleanup = ManualJobName._(r'memory-cleanup'); static const memoryCreate = ManualJobName._(r'memory-create'); static const backupDatabase = ManualJobName._(r'backup-database'); + static const integrityMissingFiles = ManualJobName._(r'integrity-missing-files'); + static const integrityOrphanFiles = ManualJobName._(r'integrity-orphan-files'); + static const integrityChecksumMismatch = ManualJobName._(r'integrity-checksum-mismatch'); + static const integrityMissingFilesRefresh = ManualJobName._(r'integrity-missing-files-refresh'); + static const integrityOrphanFilesRefresh = ManualJobName._(r'integrity-orphan-files-refresh'); + static const integrityChecksumMismatchRefresh = ManualJobName._(r'integrity-checksum-mismatch-refresh'); + static const integrityMissingFilesDeleteAll = ManualJobName._(r'integrity-missing-files-delete-all'); + static const integrityOrphanFilesDeleteAll = ManualJobName._(r'integrity-orphan-files-delete-all'); + static const integrityChecksumMismatchDeleteAll = ManualJobName._(r'integrity-checksum-mismatch-delete-all'); /// List of all possible values in this [enum][ManualJobName]. static const values = [ @@ -38,6 +47,15 @@ class ManualJobName { memoryCleanup, memoryCreate, backupDatabase, + integrityMissingFiles, + integrityOrphanFiles, + integrityChecksumMismatch, + integrityMissingFilesRefresh, + integrityOrphanFilesRefresh, + integrityChecksumMismatchRefresh, + integrityMissingFilesDeleteAll, + integrityOrphanFilesDeleteAll, + integrityChecksumMismatchDeleteAll, ]; static ManualJobName? fromJson(dynamic value) => ManualJobNameTypeTransformer().decode(value); @@ -82,6 +100,15 @@ class ManualJobNameTypeTransformer { case r'memory-cleanup': return ManualJobName.memoryCleanup; case r'memory-create': return ManualJobName.memoryCreate; case r'backup-database': return ManualJobName.backupDatabase; + case r'integrity-missing-files': return ManualJobName.integrityMissingFiles; + case r'integrity-orphan-files': return ManualJobName.integrityOrphanFiles; + case r'integrity-checksum-mismatch': return ManualJobName.integrityChecksumMismatch; + case r'integrity-missing-files-refresh': return ManualJobName.integrityMissingFilesRefresh; + case r'integrity-orphan-files-refresh': return ManualJobName.integrityOrphanFilesRefresh; + case r'integrity-checksum-mismatch-refresh': return ManualJobName.integrityChecksumMismatchRefresh; + case r'integrity-missing-files-delete-all': return ManualJobName.integrityMissingFilesDeleteAll; + case r'integrity-orphan-files-delete-all': return ManualJobName.integrityOrphanFilesDeleteAll; + case r'integrity-checksum-mismatch-delete-all': return ManualJobName.integrityChecksumMismatchDeleteAll; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/mobile/openapi/lib/model/queue_name.dart b/mobile/openapi/lib/model/queue_name.dart index bcc4159fce..2c64b93f7e 100644 --- a/mobile/openapi/lib/model/queue_name.dart +++ b/mobile/openapi/lib/model/queue_name.dart @@ -40,6 +40,7 @@ class QueueName { static const backupDatabase = QueueName._(r'backupDatabase'); static const ocr = QueueName._(r'ocr'); static const workflow = QueueName._(r'workflow'); + static const integrityCheck = QueueName._(r'integrityCheck'); /// List of all possible values in this [enum][QueueName]. static const values = [ @@ -60,6 +61,7 @@ class QueueName { backupDatabase, ocr, workflow, + integrityCheck, ]; static QueueName? fromJson(dynamic value) => QueueNameTypeTransformer().decode(value); @@ -115,6 +117,7 @@ class QueueNameTypeTransformer { case r'backupDatabase': return QueueName.backupDatabase; case r'ocr': return QueueName.ocr; case r'workflow': return QueueName.workflow; + case r'integrityCheck': return QueueName.integrityCheck; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/mobile/openapi/lib/model/queues_response_legacy_dto.dart b/mobile/openapi/lib/model/queues_response_legacy_dto.dart index 4aab6d863b..278f1bff39 100644 --- a/mobile/openapi/lib/model/queues_response_legacy_dto.dart +++ b/mobile/openapi/lib/model/queues_response_legacy_dto.dart @@ -18,6 +18,7 @@ class QueuesResponseLegacyDto { required this.duplicateDetection, required this.faceDetection, required this.facialRecognition, + required this.integrityCheck, required this.library_, required this.metadataExtraction, required this.migration, @@ -42,6 +43,8 @@ class QueuesResponseLegacyDto { QueueResponseLegacyDto facialRecognition; + QueueResponseLegacyDto integrityCheck; + QueueResponseLegacyDto library_; QueueResponseLegacyDto metadataExtraction; @@ -73,6 +76,7 @@ class QueuesResponseLegacyDto { other.duplicateDetection == duplicateDetection && other.faceDetection == faceDetection && other.facialRecognition == facialRecognition && + other.integrityCheck == integrityCheck && other.library_ == library_ && other.metadataExtraction == metadataExtraction && other.migration == migration && @@ -94,6 +98,7 @@ class QueuesResponseLegacyDto { (duplicateDetection.hashCode) + (faceDetection.hashCode) + (facialRecognition.hashCode) + + (integrityCheck.hashCode) + (library_.hashCode) + (metadataExtraction.hashCode) + (migration.hashCode) + @@ -108,7 +113,7 @@ class QueuesResponseLegacyDto { (workflow.hashCode); @override - String toString() => 'QueuesResponseLegacyDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]'; + String toString() => 'QueuesResponseLegacyDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, integrityCheck=$integrityCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]'; Map toJson() { final json = {}; @@ -117,6 +122,7 @@ class QueuesResponseLegacyDto { json[r'duplicateDetection'] = this.duplicateDetection; json[r'faceDetection'] = this.faceDetection; json[r'facialRecognition'] = this.facialRecognition; + json[r'integrityCheck'] = this.integrityCheck; json[r'library'] = this.library_; json[r'metadataExtraction'] = this.metadataExtraction; json[r'migration'] = this.migration; @@ -146,6 +152,7 @@ class QueuesResponseLegacyDto { duplicateDetection: QueueResponseLegacyDto.fromJson(json[r'duplicateDetection'])!, faceDetection: QueueResponseLegacyDto.fromJson(json[r'faceDetection'])!, facialRecognition: QueueResponseLegacyDto.fromJson(json[r'facialRecognition'])!, + integrityCheck: QueueResponseLegacyDto.fromJson(json[r'integrityCheck'])!, library_: QueueResponseLegacyDto.fromJson(json[r'library'])!, metadataExtraction: QueueResponseLegacyDto.fromJson(json[r'metadataExtraction'])!, migration: QueueResponseLegacyDto.fromJson(json[r'migration'])!, @@ -210,6 +217,7 @@ class QueuesResponseLegacyDto { 'duplicateDetection', 'faceDetection', 'facialRecognition', + 'integrityCheck', 'library', 'metadataExtraction', 'migration', diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dart index 38dbb30f0c..3a5e15b030 100644 --- a/mobile/openapi/lib/model/system_config_dto.dart +++ b/mobile/openapi/lib/model/system_config_dto.dart @@ -16,6 +16,7 @@ class SystemConfigDto { required this.backup, required this.ffmpeg, required this.image, + required this.integrityChecks, required this.job, required this.library_, required this.logging, @@ -42,6 +43,8 @@ class SystemConfigDto { SystemConfigImageDto image; + SystemConfigIntegrityChecks integrityChecks; + SystemConfigJobDto job; SystemConfigLibraryDto library_; @@ -83,6 +86,7 @@ class SystemConfigDto { other.backup == backup && other.ffmpeg == ffmpeg && other.image == image && + other.integrityChecks == integrityChecks && other.job == job && other.library_ == library_ && other.logging == logging && @@ -108,6 +112,7 @@ class SystemConfigDto { (backup.hashCode) + (ffmpeg.hashCode) + (image.hashCode) + + (integrityChecks.hashCode) + (job.hashCode) + (library_.hashCode) + (logging.hashCode) + @@ -128,13 +133,14 @@ class SystemConfigDto { (user.hashCode); @override - String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]'; + String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, integrityChecks=$integrityChecks, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]'; Map toJson() { final json = {}; json[r'backup'] = this.backup; json[r'ffmpeg'] = this.ffmpeg; json[r'image'] = this.image; + json[r'integrityChecks'] = this.integrityChecks; json[r'job'] = this.job; json[r'library'] = this.library_; json[r'logging'] = this.logging; @@ -168,6 +174,7 @@ class SystemConfigDto { backup: SystemConfigBackupsDto.fromJson(json[r'backup'])!, ffmpeg: SystemConfigFFmpegDto.fromJson(json[r'ffmpeg'])!, image: SystemConfigImageDto.fromJson(json[r'image'])!, + integrityChecks: SystemConfigIntegrityChecks.fromJson(json[r'integrityChecks'])!, job: SystemConfigJobDto.fromJson(json[r'job'])!, library_: SystemConfigLibraryDto.fromJson(json[r'library'])!, logging: SystemConfigLoggingDto.fromJson(json[r'logging'])!, @@ -236,6 +243,7 @@ class SystemConfigDto { 'backup', 'ffmpeg', 'image', + 'integrityChecks', 'job', 'library', 'logging', diff --git a/mobile/openapi/lib/model/system_config_integrity_checks.dart b/mobile/openapi/lib/model/system_config_integrity_checks.dart new file mode 100644 index 0000000000..37a75ed3d7 --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_checks.dart @@ -0,0 +1,115 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityChecks { + /// Returns a new [SystemConfigIntegrityChecks] instance. + SystemConfigIntegrityChecks({ + required this.checksumFiles, + required this.missingFiles, + required this.orphanedFiles, + }); + + SystemConfigIntegrityChecksumJob checksumFiles; + + SystemConfigIntegrityJob missingFiles; + + SystemConfigIntegrityJob orphanedFiles; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityChecks && + other.checksumFiles == checksumFiles && + other.missingFiles == missingFiles && + other.orphanedFiles == orphanedFiles; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (checksumFiles.hashCode) + + (missingFiles.hashCode) + + (orphanedFiles.hashCode); + + @override + String toString() => 'SystemConfigIntegrityChecks[checksumFiles=$checksumFiles, missingFiles=$missingFiles, orphanedFiles=$orphanedFiles]'; + + Map toJson() { + final json = {}; + json[r'checksumFiles'] = this.checksumFiles; + json[r'missingFiles'] = this.missingFiles; + json[r'orphanedFiles'] = this.orphanedFiles; + return json; + } + + /// Returns a new [SystemConfigIntegrityChecks] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityChecks? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityChecks"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityChecks( + checksumFiles: SystemConfigIntegrityChecksumJob.fromJson(json[r'checksumFiles'])!, + missingFiles: SystemConfigIntegrityJob.fromJson(json[r'missingFiles'])!, + orphanedFiles: SystemConfigIntegrityJob.fromJson(json[r'orphanedFiles'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityChecks.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityChecks.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityChecks-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityChecks.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'checksumFiles', + 'missingFiles', + 'orphanedFiles', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart b/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart new file mode 100644 index 0000000000..16b64bf2b9 --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart @@ -0,0 +1,123 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityChecksumJob { + /// Returns a new [SystemConfigIntegrityChecksumJob] instance. + SystemConfigIntegrityChecksumJob({ + required this.cronExpression, + required this.enabled, + required this.percentageLimit, + required this.timeLimit, + }); + + String cronExpression; + + bool enabled; + + num percentageLimit; + + num timeLimit; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityChecksumJob && + other.cronExpression == cronExpression && + other.enabled == enabled && + other.percentageLimit == percentageLimit && + other.timeLimit == timeLimit; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (cronExpression.hashCode) + + (enabled.hashCode) + + (percentageLimit.hashCode) + + (timeLimit.hashCode); + + @override + String toString() => 'SystemConfigIntegrityChecksumJob[cronExpression=$cronExpression, enabled=$enabled, percentageLimit=$percentageLimit, timeLimit=$timeLimit]'; + + Map toJson() { + final json = {}; + json[r'cronExpression'] = this.cronExpression; + json[r'enabled'] = this.enabled; + json[r'percentageLimit'] = this.percentageLimit; + json[r'timeLimit'] = this.timeLimit; + return json; + } + + /// Returns a new [SystemConfigIntegrityChecksumJob] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityChecksumJob? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityChecksumJob"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityChecksumJob( + cronExpression: mapValueOfType(json, r'cronExpression')!, + enabled: mapValueOfType(json, r'enabled')!, + percentageLimit: num.parse('${json[r'percentageLimit']}'), + timeLimit: num.parse('${json[r'timeLimit']}'), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityChecksumJob.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityChecksumJob.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityChecksumJob-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityChecksumJob.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'cronExpression', + 'enabled', + 'percentageLimit', + 'timeLimit', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_integrity_job.dart b/mobile/openapi/lib/model/system_config_integrity_job.dart new file mode 100644 index 0000000000..08eb559c7f --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_job.dart @@ -0,0 +1,107 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityJob { + /// Returns a new [SystemConfigIntegrityJob] instance. + SystemConfigIntegrityJob({ + required this.cronExpression, + required this.enabled, + }); + + String cronExpression; + + bool enabled; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityJob && + other.cronExpression == cronExpression && + other.enabled == enabled; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (cronExpression.hashCode) + + (enabled.hashCode); + + @override + String toString() => 'SystemConfigIntegrityJob[cronExpression=$cronExpression, enabled=$enabled]'; + + Map toJson() { + final json = {}; + json[r'cronExpression'] = this.cronExpression; + json[r'enabled'] = this.enabled; + return json; + } + + /// Returns a new [SystemConfigIntegrityJob] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityJob? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityJob"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityJob( + cronExpression: mapValueOfType(json, r'cronExpression')!, + enabled: mapValueOfType(json, r'enabled')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityJob.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityJob.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityJob-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityJob.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'cronExpression', + 'enabled', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_job_dto.dart b/mobile/openapi/lib/model/system_config_job_dto.dart index 461420b3e3..8d52d487cd 100644 --- a/mobile/openapi/lib/model/system_config_job_dto.dart +++ b/mobile/openapi/lib/model/system_config_job_dto.dart @@ -15,6 +15,7 @@ class SystemConfigJobDto { SystemConfigJobDto({ required this.backgroundTask, required this.faceDetection, + required this.integrityCheck, required this.library_, required this.metadataExtraction, required this.migration, @@ -32,6 +33,8 @@ class SystemConfigJobDto { JobSettingsDto faceDetection; + JobSettingsDto integrityCheck; + JobSettingsDto library_; JobSettingsDto metadataExtraction; @@ -58,6 +61,7 @@ class SystemConfigJobDto { bool operator ==(Object other) => identical(this, other) || other is SystemConfigJobDto && other.backgroundTask == backgroundTask && other.faceDetection == faceDetection && + other.integrityCheck == integrityCheck && other.library_ == library_ && other.metadataExtraction == metadataExtraction && other.migration == migration && @@ -75,6 +79,7 @@ class SystemConfigJobDto { // ignore: unnecessary_parenthesis (backgroundTask.hashCode) + (faceDetection.hashCode) + + (integrityCheck.hashCode) + (library_.hashCode) + (metadataExtraction.hashCode) + (migration.hashCode) + @@ -88,12 +93,13 @@ class SystemConfigJobDto { (workflow.hashCode); @override - String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, faceDetection=$faceDetection, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]'; + String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, faceDetection=$faceDetection, integrityCheck=$integrityCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]'; Map toJson() { final json = {}; json[r'backgroundTask'] = this.backgroundTask; json[r'faceDetection'] = this.faceDetection; + json[r'integrityCheck'] = this.integrityCheck; json[r'library'] = this.library_; json[r'metadataExtraction'] = this.metadataExtraction; json[r'migration'] = this.migration; @@ -119,6 +125,7 @@ class SystemConfigJobDto { return SystemConfigJobDto( backgroundTask: JobSettingsDto.fromJson(json[r'backgroundTask'])!, faceDetection: JobSettingsDto.fromJson(json[r'faceDetection'])!, + integrityCheck: JobSettingsDto.fromJson(json[r'integrityCheck'])!, library_: JobSettingsDto.fromJson(json[r'library'])!, metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!, migration: JobSettingsDto.fromJson(json[r'migration'])!, @@ -179,6 +186,7 @@ class SystemConfigJobDto { static const requiredKeys = { 'backgroundTask', 'faceDetection', + 'integrityCheck', 'library', 'metadataExtraction', 'migration', diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index c052e41a49..256b1ff843 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -322,6 +322,275 @@ "x-immich-state": "Stable" } }, + "/admin/integrity/report": { + "post": { + "description": "...", + "operationId": "getIntegrityReport", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IntegrityGetReportDto" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IntegrityReportResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get integrity report by type", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v9.9.9", + "state": "Added" + }, + { + "version": "v9.9.9", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/integrity/report/{id}": { + "delete": { + "description": "...", + "operationId": "deleteIntegrityReport", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Delete report entry and perform corresponding deletion action", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v9.9.9", + "state": "Added" + }, + { + "version": "v9.9.9", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/integrity/report/{id}/file": { + "get": { + "description": "...", + "operationId": "getIntegrityReportFile", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Download the orphan/broken file if one exists", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v9.9.9", + "state": "Added" + }, + { + "version": "v9.9.9", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/integrity/report/{type}/csv": { + "get": { + "description": "...", + "operationId": "getIntegrityReportCsv", + "parameters": [ + { + "name": "type", + "required": true, + "in": "path", + "schema": { + "$ref": "#/components/schemas/IntegrityReportType" + } + } + ], + "responses": { + "200": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Export integrity report by type as CSV", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v9.9.9", + "state": "Added" + }, + { + "version": "v9.9.9", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/integrity/summary": { + "get": { + "description": "...", + "operationId": "getIntegrityReportSummary", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IntegrityReportSummaryResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get integrity report summary", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v9.9.9", + "state": "Added" + }, + { + "version": "v9.9.9", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, "/admin/maintenance": { "post": { "description": "Put Immich into or take it out of maintenance mode", @@ -14312,6 +14581,10 @@ "name": "Faces", "description": "A face is a detected human face within an asset, which can be associated with a person. Faces are normally detected via machine learning, but can also be created via manually." }, + { + "name": "Integrity (admin)", + "description": "Endpoints for viewing and managing integrity reports." + }, { "name": "Jobs", "description": "Queues and background jobs are used for processing tasks asynchronously. Queues can be paused and resumed as needed." @@ -16589,6 +16862,85 @@ ], "type": "string" }, + "IntegrityGetReportDto": { + "properties": { + "type": { + "allOf": [ + { + "$ref": "#/components/schemas/IntegrityReportType" + } + ] + } + }, + "required": [ + "type" + ], + "type": "object" + }, + "IntegrityReportDto": { + "properties": { + "id": { + "type": "string" + }, + "path": { + "type": "string" + }, + "type": { + "allOf": [ + { + "$ref": "#/components/schemas/IntegrityReportType" + } + ] + } + }, + "required": [ + "id", + "path", + "type" + ], + "type": "object" + }, + "IntegrityReportResponseDto": { + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/IntegrityReportDto" + }, + "type": "array" + } + }, + "required": [ + "items" + ], + "type": "object" + }, + "IntegrityReportSummaryResponseDto": { + "properties": { + "checksum_mismatch": { + "type": "integer" + }, + "missing_file": { + "type": "integer" + }, + "orphan_file": { + "type": "integer" + } + }, + "required": [ + "checksum_mismatch", + "missing_file", + "orphan_file" + ], + "type": "object" + }, + "IntegrityReportType": { + "enum": [ + "orphan_file", + "missing_file", + "checksum_mismatch" + ], + "type": "string" + }, "JobCreateDto": { "properties": { "name": { @@ -16660,7 +17012,16 @@ "VersionCheck", "OcrQueueAll", "Ocr", - "WorkflowRun" + "WorkflowRun", + "IntegrityOrphanedFilesQueueAll", + "IntegrityOrphanedFiles", + "IntegrityOrphanedRefresh", + "IntegrityMissingFilesQueueAll", + "IntegrityMissingFiles", + "IntegrityMissingFilesRefresh", + "IntegrityChecksumFiles", + "IntegrityChecksumFilesRefresh", + "IntegrityReportDelete" ], "type": "string" }, @@ -16929,7 +17290,16 @@ "user-cleanup", "memory-cleanup", "memory-create", - "backup-database" + "backup-database", + "integrity-missing-files", + "integrity-orphan-files", + "integrity-checksum-mismatch", + "integrity-missing-files-refresh", + "integrity-orphan-files-refresh", + "integrity-checksum-mismatch-refresh", + "integrity-missing-files-delete-all", + "integrity-orphan-files-delete-all", + "integrity-checksum-mismatch-delete-all" ], "type": "string" }, @@ -18537,7 +18907,8 @@ "notifications", "backupDatabase", "ocr", - "workflow" + "workflow", + "integrityCheck" ], "type": "string" }, @@ -18650,6 +19021,9 @@ "facialRecognition": { "$ref": "#/components/schemas/QueueResponseLegacyDto" }, + "integrityCheck": { + "$ref": "#/components/schemas/QueueResponseLegacyDto" + }, "library": { "$ref": "#/components/schemas/QueueResponseLegacyDto" }, @@ -18693,6 +19067,7 @@ "duplicateDetection", "faceDetection", "facialRecognition", + "integrityCheck", "library", "metadataExtraction", "migration", @@ -21231,6 +21606,9 @@ "image": { "$ref": "#/components/schemas/SystemConfigImageDto" }, + "integrityChecks": { + "$ref": "#/components/schemas/SystemConfigIntegrityChecks" + }, "job": { "$ref": "#/components/schemas/SystemConfigJobDto" }, @@ -21290,6 +21668,7 @@ "backup", "ffmpeg", "image", + "integrityChecks", "job", "library", "logging", @@ -21536,6 +21915,63 @@ ], "type": "object" }, + "SystemConfigIntegrityChecks": { + "properties": { + "checksumFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityChecksumJob" + }, + "missingFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityJob" + }, + "orphanedFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityJob" + } + }, + "required": [ + "checksumFiles", + "missingFiles", + "orphanedFiles" + ], + "type": "object" + }, + "SystemConfigIntegrityChecksumJob": { + "properties": { + "cronExpression": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "percentageLimit": { + "type": "number" + }, + "timeLimit": { + "type": "number" + } + }, + "required": [ + "cronExpression", + "enabled", + "percentageLimit", + "timeLimit" + ], + "type": "object" + }, + "SystemConfigIntegrityJob": { + "properties": { + "cronExpression": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "cronExpression", + "enabled" + ], + "type": "object" + }, "SystemConfigJobDto": { "properties": { "backgroundTask": { @@ -21544,6 +21980,9 @@ "faceDetection": { "$ref": "#/components/schemas/JobSettingsDto" }, + "integrityCheck": { + "$ref": "#/components/schemas/JobSettingsDto" + }, "library": { "$ref": "#/components/schemas/JobSettingsDto" }, @@ -21581,6 +22020,7 @@ "required": [ "backgroundTask", "faceDetection", + "integrityCheck", "library", "metadataExtraction", "migration", diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 537427ff03..607ef305c6 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -40,6 +40,22 @@ export type ActivityStatisticsResponseDto = { comments: number; likes: number; }; +export type IntegrityGetReportDto = { + "type": IntegrityReportType; +}; +export type IntegrityReportDto = { + id: string; + path: string; + "type": IntegrityReportType; +}; +export type IntegrityReportResponseDto = { + items: IntegrityReportDto[]; +}; +export type IntegrityReportSummaryResponseDto = { + checksum_mismatch: number; + missing_file: number; + orphan_file: number; +}; export type SetMaintenanceModeDto = { action: MaintenanceAction; }; @@ -730,6 +746,7 @@ export type QueuesResponseLegacyDto = { duplicateDetection: QueueResponseLegacyDto; faceDetection: QueueResponseLegacyDto; facialRecognition: QueueResponseLegacyDto; + integrityCheck: QueueResponseLegacyDto; library: QueueResponseLegacyDto; metadataExtraction: QueueResponseLegacyDto; migration: QueueResponseLegacyDto; @@ -1454,12 +1471,28 @@ export type SystemConfigImageDto = { preview: SystemConfigGeneratedImageDto; thumbnail: SystemConfigGeneratedImageDto; }; +export type SystemConfigIntegrityChecksumJob = { + cronExpression: string; + enabled: boolean; + percentageLimit: number; + timeLimit: number; +}; +export type SystemConfigIntegrityJob = { + cronExpression: string; + enabled: boolean; +}; +export type SystemConfigIntegrityChecks = { + checksumFiles: SystemConfigIntegrityChecksumJob; + missingFiles: SystemConfigIntegrityJob; + orphanedFiles: SystemConfigIntegrityJob; +}; export type JobSettingsDto = { concurrency: number; }; export type SystemConfigJobDto = { backgroundTask: JobSettingsDto; faceDetection: JobSettingsDto; + integrityCheck: JobSettingsDto; library: JobSettingsDto; metadataExtraction: JobSettingsDto; migration: JobSettingsDto; @@ -1606,6 +1639,7 @@ export type SystemConfigDto = { backup: SystemConfigBackupsDto; ffmpeg: SystemConfigFFmpegDto; image: SystemConfigImageDto; + integrityChecks: SystemConfigIntegrityChecks; job: SystemConfigJobDto; library: SystemConfigLibraryDto; logging: SystemConfigLoggingDto; @@ -1850,6 +1884,69 @@ export function unlinkAllOAuthAccountsAdmin(opts?: Oazapfts.RequestOpts) { method: "POST" })); } +/** + * Get integrity report by type + */ +export function getIntegrityReport({ integrityGetReportDto }: { + integrityGetReportDto: IntegrityGetReportDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 201; + data: IntegrityReportResponseDto; + }>("/admin/integrity/report", oazapfts.json({ + ...opts, + method: "POST", + body: integrityGetReportDto + }))); +} +/** + * Delete report entry and perform corresponding deletion action + */ +export function deleteIntegrityReport({ id }: { + id: string; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText(`/admin/integrity/report/${encodeURIComponent(id)}`, { + ...opts, + method: "DELETE" + })); +} +/** + * Download the orphan/broken file if one exists + */ +export function getIntegrityReportFile({ id }: { + id: string; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchBlob<{ + status: 200; + data: Blob; + }>(`/admin/integrity/report/${encodeURIComponent(id)}/file`, { + ...opts + })); +} +/** + * Export integrity report by type as CSV + */ +export function getIntegrityReportCsv({ $type }: { + $type: IntegrityReportType; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchBlob<{ + status: 200; + data: Blob; + }>(`/admin/integrity/report/${encodeURIComponent($type)}/csv`, { + ...opts + })); +} +/** + * Get integrity report summary + */ +export function getIntegrityReportSummary(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: IntegrityReportSummaryResponseDto; + }>("/admin/integrity/summary", { + ...opts + })); +} /** * Set maintenance mode */ @@ -5138,6 +5235,11 @@ export enum UserAvatarColor { Gray = "gray", Amber = "amber" } +export enum IntegrityReportType { + OrphanFile = "orphan_file", + MissingFile = "missing_file", + ChecksumMismatch = "checksum_mismatch" +} export enum MaintenanceAction { Start = "start", End = "end" @@ -5378,7 +5480,16 @@ export enum ManualJobName { UserCleanup = "user-cleanup", MemoryCleanup = "memory-cleanup", MemoryCreate = "memory-create", - BackupDatabase = "backup-database" + BackupDatabase = "backup-database", + IntegrityMissingFiles = "integrity-missing-files", + IntegrityOrphanFiles = "integrity-orphan-files", + IntegrityChecksumMismatch = "integrity-checksum-mismatch", + IntegrityMissingFilesRefresh = "integrity-missing-files-refresh", + IntegrityOrphanFilesRefresh = "integrity-orphan-files-refresh", + IntegrityChecksumMismatchRefresh = "integrity-checksum-mismatch-refresh", + IntegrityMissingFilesDeleteAll = "integrity-missing-files-delete-all", + IntegrityOrphanFilesDeleteAll = "integrity-orphan-files-delete-all", + IntegrityChecksumMismatchDeleteAll = "integrity-checksum-mismatch-delete-all" } export enum QueueName { ThumbnailGeneration = "thumbnailGeneration", @@ -5397,7 +5508,8 @@ export enum QueueName { Notifications = "notifications", BackupDatabase = "backupDatabase", Ocr = "ocr", - Workflow = "workflow" + Workflow = "workflow", + IntegrityCheck = "integrityCheck" } export enum QueueCommand { Start = "start", @@ -5486,7 +5598,16 @@ export enum JobName { VersionCheck = "VersionCheck", OcrQueueAll = "OcrQueueAll", Ocr = "Ocr", - WorkflowRun = "WorkflowRun" + WorkflowRun = "WorkflowRun", + IntegrityOrphanedFilesQueueAll = "IntegrityOrphanedFilesQueueAll", + IntegrityOrphanedFiles = "IntegrityOrphanedFiles", + IntegrityOrphanedRefresh = "IntegrityOrphanedRefresh", + IntegrityMissingFilesQueueAll = "IntegrityMissingFilesQueueAll", + IntegrityMissingFiles = "IntegrityMissingFiles", + IntegrityMissingFilesRefresh = "IntegrityMissingFilesRefresh", + IntegrityChecksumFiles = "IntegrityChecksumFiles", + IntegrityChecksumFilesRefresh = "IntegrityChecksumFilesRefresh", + IntegrityReportDelete = "IntegrityReportDelete" } export enum SearchSuggestionType { Country = "country", diff --git a/server/src/config.ts b/server/src/config.ts index c18acd79f8..4f366737bf 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -46,6 +46,22 @@ export interface SystemConfig { accelDecode: boolean; tonemap: ToneMapping; }; + integrityChecks: { + missingFiles: { + enabled: boolean; + cronExpression: string; + }; + orphanedFiles: { + enabled: boolean; + cronExpression: string; + }; + checksumFiles: { + enabled: boolean; + cronExpression: string; + timeLimit: number; + percentageLimit: number; + }; + }; job: Record; logging: { enabled: boolean; @@ -222,6 +238,22 @@ export const defaults = Object.freeze({ accel: TranscodeHardwareAcceleration.Disabled, accelDecode: false, }, + integrityChecks: { + missingFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + }, + orphanedFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + }, + checksumFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + timeLimit: 60 * 60 * 1000, // 1 hour + percentageLimit: 1, // 100% of assets + }, + }, job: { [QueueName.BackgroundTask]: { concurrency: 5 }, [QueueName.SmartSearch]: { concurrency: 2 }, @@ -236,6 +268,7 @@ export const defaults = Object.freeze({ [QueueName.Notification]: { concurrency: 5 }, [QueueName.Ocr]: { concurrency: 1 }, [QueueName.Workflow]: { concurrency: 5 }, + [QueueName.IntegrityCheck]: { concurrency: 1 }, }, logging: { enabled: true, diff --git a/server/src/constants.ts b/server/src/constants.ts index 33f8e3b4c5..f6c541317c 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -146,6 +146,7 @@ export const endpointTags: Record = { [ApiTag.Duplicates]: 'Endpoints for managing and identifying duplicate assets.', [ApiTag.Faces]: 'A face is a detected human face within an asset, which can be associated with a person. Faces are normally detected via machine learning, but can also be created via manually.', + [ApiTag.Integrity]: 'Endpoints for viewing and managing integrity reports.', [ApiTag.Jobs]: 'Queues and background jobs are used for processing tasks asynchronously. Queues can be paused and resumed as needed.', [ApiTag.Libraries]: diff --git a/server/src/controllers/index.ts b/server/src/controllers/index.ts index 6ba3d38a73..5b407f0ac9 100644 --- a/server/src/controllers/index.ts +++ b/server/src/controllers/index.ts @@ -9,6 +9,7 @@ import { AuthController } from 'src/controllers/auth.controller'; import { DownloadController } from 'src/controllers/download.controller'; import { DuplicateController } from 'src/controllers/duplicate.controller'; import { FaceController } from 'src/controllers/face.controller'; +import { IntegrityController } from 'src/controllers/integrity.controller'; import { JobController } from 'src/controllers/job.controller'; import { LibraryController } from 'src/controllers/library.controller'; import { MaintenanceController } from 'src/controllers/maintenance.controller'; @@ -49,6 +50,7 @@ export const controllers = [ DownloadController, DuplicateController, FaceController, + IntegrityController, JobController, LibraryController, MaintenanceController, diff --git a/server/src/controllers/integrity.controller.ts b/server/src/controllers/integrity.controller.ts new file mode 100644 index 0000000000..84299899ab --- /dev/null +++ b/server/src/controllers/integrity.controller.ts @@ -0,0 +1,90 @@ +import { Body, Controller, Delete, Get, Next, Param, Post, Res } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; +import { NextFunction, Response } from 'express'; +import { Endpoint, HistoryBuilder } from 'src/decorators'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { + IntegrityGetReportDto, + IntegrityReportResponseDto, + IntegrityReportSummaryResponseDto, +} from 'src/dtos/integrity.dto'; +import { ApiTag, Permission } from 'src/enum'; +import { Auth, Authenticated, FileResponse } from 'src/middleware/auth.guard'; +import { LoggingRepository } from 'src/repositories/logging.repository'; +import { IntegrityService } from 'src/services/integrity.service'; +import { sendFile } from 'src/utils/file'; +import { IntegrityReportTypeParamDto, UUIDParamDto } from 'src/validation'; + +@ApiTags(ApiTag.Maintenance) +@Controller('admin/integrity') +export class IntegrityController { + constructor( + private logger: LoggingRepository, + private service: IntegrityService, + ) {} + + @Get('summary') + @Endpoint({ + summary: 'Get integrity report summary', + description: '...', + history: new HistoryBuilder().added('v9.9.9').alpha('v9.9.9'), + }) + @Authenticated({ permission: Permission.Maintenance, admin: true }) + getIntegrityReportSummary(): Promise { + return this.service.getIntegrityReportSummary(); + } + + @Post('report') + @Endpoint({ + summary: 'Get integrity report by type', + description: '...', + history: new HistoryBuilder().added('v9.9.9').alpha('v9.9.9'), + }) + @Authenticated({ permission: Permission.Maintenance, admin: true }) + getIntegrityReport(@Body() dto: IntegrityGetReportDto): Promise { + return this.service.getIntegrityReport(dto); + } + + @Delete('report/:id') + @Endpoint({ + summary: 'Delete report entry and perform corresponding deletion action', + description: '...', + history: new HistoryBuilder().added('v9.9.9').alpha('v9.9.9'), + }) + @Authenticated({ permission: Permission.Maintenance, admin: true }) + async deleteIntegrityReport(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto): Promise { + await this.service.deleteIntegrityReport(auth, id); + } + + @Get('report/:type/csv') + @Endpoint({ + summary: 'Export integrity report by type as CSV', + description: '...', + history: new HistoryBuilder().added('v9.9.9').alpha('v9.9.9'), + }) + @FileResponse() + @Authenticated({ permission: Permission.Maintenance, admin: true }) + getIntegrityReportCsv(@Param() { type }: IntegrityReportTypeParamDto, @Res() res: Response): void { + res.setHeader('Content-Type', 'text/csv'); + res.setHeader('Cache-Control', 'private, no-cache, no-transform'); + res.setHeader('Content-Disposition', `attachment; filename="${encodeURIComponent(`${Date.now()}-${type}.csv`)}"`); + + this.service.getIntegrityReportCsv(type).pipe(res); + } + + @Get('report/:id/file') + @Endpoint({ + summary: 'Download the orphan/broken file if one exists', + description: '...', + history: new HistoryBuilder().added('v9.9.9').alpha('v9.9.9'), + }) + @FileResponse() + @Authenticated({ permission: Permission.Maintenance, admin: true }) + async getIntegrityReportFile( + @Param() { id }: UUIDParamDto, + @Res() res: Response, + @Next() next: NextFunction, + ): Promise { + await sendFile(res, next, () => this.service.getIntegrityReportFile(id), this.logger); + } +} diff --git a/server/src/dtos/integrity.dto.ts b/server/src/dtos/integrity.dto.ts new file mode 100644 index 0000000000..9a51adbca0 --- /dev/null +++ b/server/src/dtos/integrity.dto.ts @@ -0,0 +1,40 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IntegrityReportType } from 'src/enum'; +import { ValidateEnum } from 'src/validation'; + +export class IntegrityReportSummaryResponseDto { + @ApiProperty({ type: 'integer' }) + [IntegrityReportType.ChecksumFail]!: number; + @ApiProperty({ type: 'integer' }) + [IntegrityReportType.MissingFile]!: number; + @ApiProperty({ type: 'integer' }) + [IntegrityReportType.OrphanFile]!: number; +} + +export class IntegrityGetReportDto { + @ValidateEnum({ enum: IntegrityReportType, name: 'IntegrityReportType' }) + type!: IntegrityReportType; + + // todo: paginate + // @IsInt() + // @Min(1) + // @Type(() => Number) + // @Optional() + // page?: number; +} + +export class IntegrityDeleteReportDto { + @ValidateEnum({ enum: IntegrityReportType, name: 'IntegrityReportType' }) + type!: IntegrityReportType; +} + +class IntegrityReportDto { + id!: string; + @ValidateEnum({ enum: IntegrityReportType, name: 'IntegrityReportType' }) + type!: IntegrityReportType; + path!: string; +} + +export class IntegrityReportResponseDto { + items!: IntegrityReportDto[]; +} diff --git a/server/src/dtos/queue-legacy.dto.ts b/server/src/dtos/queue-legacy.dto.ts index 79155e3f74..1f08ccbb93 100644 --- a/server/src/dtos/queue-legacy.dto.ts +++ b/server/src/dtos/queue-legacy.dto.ts @@ -66,6 +66,9 @@ export class QueuesResponseLegacyDto implements Record { diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index c835073c31..2510c8768e 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -38,6 +38,7 @@ const isOAuthEnabled = (config: SystemConfigOAuthDto) => config.enabled; const isOAuthOverrideEnabled = (config: SystemConfigOAuthDto) => config.mobileOverrideEnabled; const isEmailNotificationEnabled = (config: SystemConfigSmtpDto) => config.enabled; const isDatabaseBackupEnabled = (config: DatabaseBackupConfig) => config.enabled; +const isEnabledProperty = (config: { enabled: boolean }) => config.enabled; export class DatabaseBackupConfig { @ValidateBoolean() @@ -145,6 +146,42 @@ export class SystemConfigFFmpegDto { tonemap!: ToneMapping; } +class SystemConfigIntegrityJob { + @ValidateBoolean() + enabled!: boolean; + + @ValidateIf(isEnabledProperty) + @IsNotEmpty() + @IsCronExpression() + @IsString() + cronExpression!: string; +} + +class SystemConfigIntegrityChecksumJob extends SystemConfigIntegrityJob { + @IsInt() + timeLimit!: number; + + @IsNumber() + percentageLimit!: number; +} + +class SystemConfigIntegrityChecks { + @Type(() => SystemConfigIntegrityJob) + @ValidateNested() + @IsObject() + missingFiles!: SystemConfigIntegrityJob; + + @Type(() => SystemConfigIntegrityJob) + @ValidateNested() + @IsObject() + orphanedFiles!: SystemConfigIntegrityJob; + + @Type(() => SystemConfigIntegrityChecksumJob) + @ValidateNested() + @IsObject() + checksumFiles!: SystemConfigIntegrityChecksumJob; +} + class JobSettingsDto { @IsInt() @IsPositive() @@ -230,6 +267,12 @@ class SystemConfigJobDto implements Record @IsObject() @Type(() => JobSettingsDto) [QueueName.Workflow]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.IntegrityCheck]!: JobSettingsDto; } class SystemConfigLibraryScanDto { @@ -649,6 +692,11 @@ export class SystemConfigDto implements SystemConfig { @IsObject() ffmpeg!: SystemConfigFFmpegDto; + @Type(() => SystemConfigIntegrityChecks) + @ValidateNested() + @IsObject() + integrityChecks!: SystemConfigIntegrityChecks; + @Type(() => SystemConfigLoggingDto) @ValidateNested() @IsObject() diff --git a/server/src/enum.ts b/server/src/enum.ts index 9d0a2c0426..d70427fe52 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -302,6 +302,7 @@ export enum SystemMetadataKey { SystemFlags = 'system-flags', VersionCheckState = 'version-check-state', License = 'license', + IntegrityChecksumCheckpoint = 'integrity-checksum-checkpoint', } export enum UserMetadataKey { @@ -345,6 +346,12 @@ export enum SourceType { Manual = 'manual', } +export enum IntegrityReportType { + OrphanFile = 'orphan_file', + MissingFile = 'missing_file', + ChecksumFail = 'checksum_mismatch', +} + export enum ManualJobName { PersonCleanup = 'person-cleanup', TagCleanup = 'tag-cleanup', @@ -352,6 +359,15 @@ export enum ManualJobName { MemoryCleanup = 'memory-cleanup', MemoryCreate = 'memory-create', BackupDatabase = 'backup-database', + IntegrityMissingFiles = `integrity-missing-files`, + IntegrityOrphanFiles = `integrity-orphan-files`, + IntegrityChecksumFiles = `integrity-checksum-mismatch`, + IntegrityMissingFilesRefresh = `integrity-missing-files-refresh`, + IntegrityOrphanFilesRefresh = `integrity-orphan-files-refresh`, + IntegrityChecksumFilesRefresh = `integrity-checksum-mismatch-refresh`, + IntegrityMissingFilesDeleteAll = `integrity-missing-files-delete-all`, + IntegrityOrphanFilesDeleteAll = `integrity-orphan-files-delete-all`, + IntegrityChecksumFilesDeleteAll = `integrity-checksum-mismatch-delete-all`, } export enum AssetPathType { @@ -550,6 +566,7 @@ export enum QueueName { BackupDatabase = 'backupDatabase', Ocr = 'ocr', Workflow = 'workflow', + IntegrityCheck = 'integrityCheck', } export enum QueueJobStatus { @@ -638,6 +655,17 @@ export enum JobName { // Workflow WorkflowRun = 'WorkflowRun', + + // Integrity + IntegrityOrphanedFilesQueueAll = 'IntegrityOrphanedFilesQueueAll', + IntegrityOrphanedFiles = 'IntegrityOrphanedFiles', + IntegrityOrphanedFilesRefresh = 'IntegrityOrphanedRefresh', + IntegrityMissingFilesQueueAll = 'IntegrityMissingFilesQueueAll', + IntegrityMissingFiles = 'IntegrityMissingFiles', + IntegrityMissingFilesRefresh = 'IntegrityMissingFilesRefresh', + IntegrityChecksumFiles = 'IntegrityChecksumFiles', + IntegrityChecksumFilesRefresh = 'IntegrityChecksumFilesRefresh', + IntegrityReportDelete = 'IntegrityReportDelete', } export enum QueueCommand { @@ -680,6 +708,7 @@ export enum DatabaseLock { GetSystemConfig = 69, BackupDatabase = 42, MemoryCreation = 777, + IntegrityCheck = 67, } export enum MaintenanceAction { @@ -835,6 +864,7 @@ export enum ApiTag { Download = 'Download', Duplicates = 'Duplicates', Faces = 'Faces', + Integrity = 'Integrity (admin)', Jobs = 'Jobs', Libraries = 'Libraries', Maintenance = 'Maintenance (admin)', diff --git a/server/src/queries/integrity.repository.sql b/server/src/queries/integrity.repository.sql new file mode 100644 index 0000000000..61ece2260d --- /dev/null +++ b/server/src/queries/integrity.repository.sql @@ -0,0 +1,179 @@ +-- NOTE: This file is auto generated by ./sql-generator + +-- IntegrityRepository.getById +select + "integrity_report".* +from + "integrity_report" +where + "id" = $1 + +-- IntegrityRepository.getIntegrityReportSummary +select + count(*) filter ( + where + "type" = $1 + ) as "checksum_mismatch", + count(*) filter ( + where + "type" = $2 + ) as "missing_file", + count(*) filter ( + where + "type" = $3 + ) as "orphan_file" +from + "integrity_report" + +-- IntegrityRepository.getIntegrityReports +select + "id", + "type", + "path", + "assetId", + "fileAssetId" +from + "integrity_report" +where + "type" = $1 +order by + "createdAt" desc + +-- IntegrityRepository.getAssetPathsByPaths +select + "originalPath", + "encodedVideoPath" +from + "asset" +where + ( + "originalPath" in $1 + or "encodedVideoPath" in $2 + ) + +-- IntegrityRepository.getAssetFilePathsByPaths +select + "path" +from + "asset_file" +where + "path" in $1 + +-- IntegrityRepository.getAssetCount +select + count(*) as "count" +from + "asset" + +-- IntegrityRepository.streamAllAssetPaths +select + "id", + "type", + "path", + "assetId", + "fileAssetId" +from + "integrity_report" +where + "type" = $1 +order by + "createdAt" desc +select + "originalPath", + "encodedVideoPath" +from + "asset" + +-- IntegrityRepository.streamAllAssetFilePaths +select + "path" +from + "asset_file" + +-- IntegrityRepository.streamAssetPaths +select + "allPaths"."path" as "path", + "allPaths"."assetId", + "allPaths"."fileAssetId", + "integrity_report"."id" as "reportId" +from + ( + select + "asset"."originalPath" as "path", + "asset"."id" as "assetId", + null::uuid as "fileAssetId" + from + "asset" + where + "asset"."deletedAt" is null + union all + select + "asset"."encodedVideoPath" as "path", + "asset"."id" as "assetId", + null::uuid as "fileAssetId" + from + "asset" + where + "asset"."deletedAt" is null + and "asset"."encodedVideoPath" is not null + and "asset"."encodedVideoPath" != '' + union all + select + "path", + null::uuid as "assetId", + "asset_file"."id" as "fileAssetId" + from + "asset_file" + ) as "allPaths" + left join "integrity_report" on "integrity_report"."type" = $1 + and ( + "integrity_report"."assetId" = "allPaths"."assetId" + or "integrity_report"."fileAssetId" = "allPaths"."fileAssetId" + ) + +-- IntegrityRepository.streamAssetChecksums +select + "asset"."originalPath", + "asset"."checksum", + "asset"."createdAt", + "asset"."id" as "assetId", + "integrity_report"."id" as "reportId" +from + "asset" + left join "integrity_report" on "integrity_report"."assetId" = "asset"."id" + and "integrity_report"."type" = $1 +where + "createdAt" >= $2 + and "createdAt" <= $3 +order by + "createdAt" asc + +-- IntegrityRepository.streamIntegrityReports +select + "integrity_report"."id" as "reportId", + "integrity_report"."path" +from + "integrity_report" +where + "integrity_report"."type" = $1 + +-- IntegrityRepository.streamIntegrityReportsByProperty +select + "id", + "path", + "assetId", + "fileAssetId" +from + "integrity_report" +where + "abcdefghi" is not null + +-- IntegrityRepository.deleteById +delete from "integrity_report" +where + "id" = $1 + +-- IntegrityRepository.deleteByIds +delete from "integrity_report" +where + "id" in $1 diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index c59110d674..d89e944b64 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -15,6 +15,7 @@ import { DownloadRepository } from 'src/repositories/download.repository'; import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityRepository } from 'src/repositories/integrity.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -68,6 +69,7 @@ export const repositories = [ DuplicateRepository, EmailRepository, EventRepository, + IntegrityRepository, JobRepository, LibraryRepository, LoggingRepository, diff --git a/server/src/repositories/integrity.repository.ts b/server/src/repositories/integrity.repository.ts new file mode 100644 index 0000000000..437095a98a --- /dev/null +++ b/server/src/repositories/integrity.repository.ts @@ -0,0 +1,234 @@ +import { Injectable } from '@nestjs/common'; +import { Insertable, Kysely, sql } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; +import { Readable } from 'node:stream'; +import { DummyValue, GenerateSql } from 'src/decorators'; +import { IntegrityReportType } from 'src/enum'; +import { DB } from 'src/schema'; +import { IntegrityReportTable } from 'src/schema/tables/integrity-report.table'; + +@Injectable() +export class IntegrityRepository { + constructor(@InjectKysely() private db: Kysely) {} + + create(dto: Insertable | Insertable[]) { + return this.db + .insertInto('integrity_report') + .values(dto) + .onConflict((oc) => + oc.columns(['path', 'type']).doUpdateSet({ + assetId: (eb) => eb.ref('excluded.assetId'), + fileAssetId: (eb) => eb.ref('excluded.fileAssetId'), + }), + ) + .returningAll() + .executeTakeFirst(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getById(id: string) { + return this.db + .selectFrom('integrity_report') + .selectAll('integrity_report') + .where('id', '=', id) + .executeTakeFirstOrThrow(); + } + + @GenerateSql({ params: [] }) + getIntegrityReportSummary() { + return this.db + .selectFrom('integrity_report') + .select((eb) => + eb.fn + .countAll() + .filterWhere('type', '=', IntegrityReportType.ChecksumFail) + .as(IntegrityReportType.ChecksumFail), + ) + .select((eb) => + eb.fn + .countAll() + .filterWhere('type', '=', IntegrityReportType.MissingFile) + .as(IntegrityReportType.MissingFile), + ) + .select((eb) => + eb.fn + .countAll() + .filterWhere('type', '=', IntegrityReportType.OrphanFile) + .as(IntegrityReportType.OrphanFile), + ) + .executeTakeFirstOrThrow(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getIntegrityReports(type: IntegrityReportType) { + return this.db + .selectFrom('integrity_report') + .select(['id', 'type', 'path', 'assetId', 'fileAssetId']) + .where('type', '=', type) + .orderBy('createdAt', 'desc') + .execute(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getAssetPathsByPaths(paths: string[]) { + return this.db + .selectFrom('asset') + .select(['originalPath', 'encodedVideoPath']) + .where((eb) => eb.or([eb('originalPath', 'in', paths), eb('encodedVideoPath', 'in', paths)])) + .execute(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getAssetFilePathsByPaths(paths: string[]) { + return this.db.selectFrom('asset_file').select(['path']).where('path', 'in', paths).execute(); + } + + @GenerateSql({ params: [] }) + getAssetCount() { + return this.db + .selectFrom('asset') + .select((eb) => eb.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + } + + @GenerateSql({ params: [DummyValue.STRING], stream: true }) + streamIntegrityReportsCSV(type: IntegrityReportType): Readable { + const items = this.db + .selectFrom('integrity_report') + .select(['id', 'type', 'path', 'assetId', 'fileAssetId']) + .where('type', '=', type) + .orderBy('createdAt', 'desc') + .stream(); + + // very rudimentary csv serialiser + async function* generator() { + yield 'id,type,assetId,fileAssetId,path\n'; + + for await (const item of items) { + // no expectation of particularly bad filenames + // but they could potentially have a newline or quote character + yield `${item.id},${item.type},${item.assetId},${item.fileAssetId},"${item.path.replaceAll('"', '""')}"\n`; + } + } + + return Readable.from(generator()); + } + + @GenerateSql({ params: [], stream: true }) + streamAllAssetPaths() { + return this.db.selectFrom('asset').select(['originalPath', 'encodedVideoPath']).stream(); + } + + @GenerateSql({ params: [], stream: true }) + streamAllAssetFilePaths() { + return this.db.selectFrom('asset_file').select(['path']).stream(); + } + + @GenerateSql({ params: [], stream: true }) + streamAssetPaths() { + return this.db + .selectFrom((eb) => + eb + .selectFrom('asset') + .where('asset.deletedAt', 'is', null) + .select(['asset.originalPath as path']) + .select((eb) => [ + eb.ref('asset.id').$castTo().as('assetId'), + sql`null::uuid`.as('fileAssetId'), + ]) + .unionAll( + eb + .selectFrom('asset') + .where('asset.deletedAt', 'is', null) + .select((eb) => [ + eb.ref('asset.encodedVideoPath').$castTo().as('path'), + eb.ref('asset.id').$castTo().as('assetId'), + sql`null::uuid`.as('fileAssetId'), + ]) + .where('asset.encodedVideoPath', 'is not', null) + .where('asset.encodedVideoPath', '!=', sql`''`), + ) + .unionAll( + eb + .selectFrom('asset_file') + .select(['path']) + .select((eb) => [ + sql`null::uuid`.as('assetId'), + eb.ref('asset_file.id').$castTo().as('fileAssetId'), + ]), + ) + .as('allPaths'), + ) + .leftJoin( + 'integrity_report', + (join) => + join + .on('integrity_report.type', '=', IntegrityReportType.OrphanFile) + .on((eb) => + eb.or([ + eb('integrity_report.assetId', '=', eb.ref('allPaths.assetId')), + eb('integrity_report.fileAssetId', '=', eb.ref('allPaths.fileAssetId')), + ]), + ), + // .onRef('integrity_report.path', '=', 'allPaths.path') + ) + .select(['allPaths.path as path', 'allPaths.assetId', 'allPaths.fileAssetId', 'integrity_report.id as reportId']) + .stream(); + } + + @GenerateSql({ params: [DummyValue.DATE, DummyValue.DATE], stream: true }) + streamAssetChecksums(startMarker?: Date, endMarker?: Date) { + return this.db + .selectFrom('asset') + .leftJoin('integrity_report', (join) => + join + .onRef('integrity_report.assetId', '=', 'asset.id') + // .onRef('integrity_report.path', '=', 'asset.originalPath') + .on('integrity_report.type', '=', IntegrityReportType.ChecksumFail), + ) + .select([ + 'asset.originalPath', + 'asset.checksum', + 'asset.createdAt', + 'asset.id as assetId', + 'integrity_report.id as reportId', + ]) + .$if(startMarker !== undefined, (qb) => qb.where('createdAt', '>=', startMarker!)) + .$if(endMarker !== undefined, (qb) => qb.where('createdAt', '<=', endMarker!)) + .orderBy('createdAt', 'asc') + .stream(); + } + + @GenerateSql({ params: [DummyValue.STRING], stream: true }) + streamIntegrityReports(type: IntegrityReportType) { + return this.db + .selectFrom('integrity_report') + .select(['integrity_report.id as reportId', 'integrity_report.path']) + .where('integrity_report.type', '=', type) + .$if(type === IntegrityReportType.ChecksumFail, (eb) => + eb.leftJoin('asset', 'integrity_report.path', 'asset.originalPath').select('asset.checksum'), + ) + .stream(); + } + + @GenerateSql({ params: [DummyValue.STRING], stream: true }) + streamIntegrityReportsByProperty(property?: 'assetId' | 'fileAssetId', filterType?: IntegrityReportType) { + return this.db + .selectFrom('integrity_report') + .select(['id', 'path', 'assetId', 'fileAssetId']) + .$if(filterType !== undefined, (eb) => eb.where('type', '=', filterType!)) + .$if(property === undefined, (eb) => eb.where('assetId', 'is', null).where('fileAssetId', 'is', null)) + .$if(property !== undefined, (eb) => eb.where(property!, 'is not', null)) + .stream(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + deleteById(id: string) { + return this.db.deleteFrom('integrity_report').where('id', '=', id).execute(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + deleteByIds(ids: string[]) { + return this.db.deleteFrom('integrity_report').where('id', 'in', ids).execute(); + } +} diff --git a/server/src/schema/index.ts b/server/src/schema/index.ts index 9e206826e6..cea30cded1 100644 --- a/server/src/schema/index.ts +++ b/server/src/schema/index.ts @@ -40,6 +40,7 @@ import { AssetTable } from 'src/schema/tables/asset.table'; import { AuditTable } from 'src/schema/tables/audit.table'; import { FaceSearchTable } from 'src/schema/tables/face-search.table'; import { GeodataPlacesTable } from 'src/schema/tables/geodata-places.table'; +import { IntegrityReportTable } from 'src/schema/tables/integrity-report.table'; import { LibraryTable } from 'src/schema/tables/library.table'; import { MemoryAssetAuditTable } from 'src/schema/tables/memory-asset-audit.table'; import { MemoryAssetTable } from 'src/schema/tables/memory-asset.table'; @@ -98,6 +99,7 @@ export class ImmichDatabase { AssetExifTable, FaceSearchTable, GeodataPlacesTable, + IntegrityReportTable, LibraryTable, MemoryTable, MemoryAuditTable, @@ -195,6 +197,8 @@ export interface DB { geodata_places: GeodataPlacesTable; + integrity_report: IntegrityReportTable; + library: LibraryTable; memory: MemoryTable; diff --git a/server/src/schema/migrations/1764255490085-CreateIntegrityReportTable.ts b/server/src/schema/migrations/1764255490085-CreateIntegrityReportTable.ts new file mode 100644 index 0000000000..bbff9184a1 --- /dev/null +++ b/server/src/schema/migrations/1764255490085-CreateIntegrityReportTable.ts @@ -0,0 +1,22 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql`CREATE TABLE "integrity_report" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4(), + "type" character varying NOT NULL, + "path" character varying NOT NULL, + "createdAt" timestamp with time zone NOT NULL DEFAULT now(), + "assetId" uuid, + "fileAssetId" uuid, + CONSTRAINT "integrity_report_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT "integrity_report_fileAssetId_fkey" FOREIGN KEY ("fileAssetId") REFERENCES "asset_file" ("id") ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT "integrity_report_type_path_uq" UNIQUE ("type", "path"), + CONSTRAINT "integrity_report_pkey" PRIMARY KEY ("id") +);`.execute(db); + await sql`CREATE INDEX "integrity_report_assetId_idx" ON "integrity_report" ("assetId");`.execute(db); + await sql`CREATE INDEX "integrity_report_fileAssetId_idx" ON "integrity_report" ("fileAssetId");`.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`DROP TABLE "integrity_report";`.execute(db); +} diff --git a/server/src/schema/tables/integrity-report.table.ts b/server/src/schema/tables/integrity-report.table.ts new file mode 100644 index 0000000000..34ae50ab8e --- /dev/null +++ b/server/src/schema/tables/integrity-report.table.ts @@ -0,0 +1,35 @@ +import { IntegrityReportType } from 'src/enum'; +import { AssetFileTable } from 'src/schema/tables/asset-file.table'; +import { AssetTable } from 'src/schema/tables/asset.table'; +import { + Column, + CreateDateColumn, + ForeignKeyColumn, + Generated, + PrimaryGeneratedColumn, + Table, + Timestamp, + Unique, +} from 'src/sql-tools'; + +@Table('integrity_report') +@Unique({ columns: ['type', 'path'] }) +export class IntegrityReportTable { + @PrimaryGeneratedColumn() + id!: Generated; + + @Column() + type!: IntegrityReportType; + + @Column() + path!: string; + + @CreateDateColumn() + createdAt!: Generated; + + @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: true }) + assetId!: string | null; + + @ForeignKeyColumn(() => AssetFileTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: true }) + fileAssetId!: string | null; +} diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index 9c422818b3..3dfbfa6bd0 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -22,6 +22,7 @@ import { DownloadRepository } from 'src/repositories/download.repository'; import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityRepository } from 'src/repositories/integrity.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -79,6 +80,7 @@ export const BASE_SERVICE_DEPENDENCIES = [ DuplicateRepository, EmailRepository, EventRepository, + IntegrityRepository, JobRepository, LibraryRepository, MachineLearningRepository, @@ -137,6 +139,7 @@ export class BaseService { protected duplicateRepository: DuplicateRepository, protected emailRepository: EmailRepository, protected eventRepository: EventRepository, + protected integrityRepository: IntegrityRepository, protected jobRepository: JobRepository, protected libraryRepository: LibraryRepository, protected machineLearningRepository: MachineLearningRepository, diff --git a/server/src/services/index.ts b/server/src/services/index.ts index eeb8424048..42bf467fd0 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -12,6 +12,7 @@ import { CliService } from 'src/services/cli.service'; import { DatabaseService } from 'src/services/database.service'; import { DownloadService } from 'src/services/download.service'; import { DuplicateService } from 'src/services/duplicate.service'; +import { IntegrityService } from 'src/services/integrity.service'; import { JobService } from 'src/services/job.service'; import { LibraryService } from 'src/services/library.service'; import { MaintenanceService } from 'src/services/maintenance.service'; @@ -62,6 +63,7 @@ export const services = [ DatabaseService, DownloadService, DuplicateService, + IntegrityService, JobService, LibraryService, MaintenanceService, diff --git a/server/src/services/integrity.service.spec.ts b/server/src/services/integrity.service.spec.ts new file mode 100644 index 0000000000..7266928565 --- /dev/null +++ b/server/src/services/integrity.service.spec.ts @@ -0,0 +1,24 @@ +import { IntegrityService } from 'src/services/integrity.service'; +import { newTestService, ServiceMocks } from 'test/utils'; + +describe(IntegrityService.name, () => { + let sut: IntegrityService; + // impl. pending + // eslint-disable-next-line @typescript-eslint/no-unused-vars + let mocks: ServiceMocks; + + beforeEach(() => { + ({ sut, mocks } = newTestService(IntegrityService)); + }); + + it('should work', () => { + expect(sut).toBeDefined(); + }); + + describe.skip('getIntegrityReportSummary'); // just calls repository + describe.skip('getIntegrityReport'); // just calls repository + describe.skip('getIntegrityReportCsv'); // just calls repository + + describe.todo('getIntegrityReportFile'); + describe.todo('deleteIntegrityReport'); +}); diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts new file mode 100644 index 0000000000..56d360ac2f --- /dev/null +++ b/server/src/services/integrity.service.ts @@ -0,0 +1,688 @@ +import { Injectable } from '@nestjs/common'; +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { basename } from 'node:path'; +import { Readable, Writable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; +import { StorageCore } from 'src/cores/storage.core'; +import { OnEvent, OnJob } from 'src/decorators'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { + IntegrityGetReportDto, + IntegrityReportResponseDto, + IntegrityReportSummaryResponseDto, +} from 'src/dtos/integrity.dto'; +import { + AssetStatus, + CacheControl, + DatabaseLock, + ImmichWorker, + IntegrityReportType, + JobName, + JobStatus, + QueueName, + StorageFolder, + SystemMetadataKey, +} from 'src/enum'; +import { ArgOf } from 'src/repositories/event.repository'; +import { BaseService } from 'src/services/base.service'; +import { + IIntegrityDeleteReportJob, + IIntegrityJob, + IIntegrityMissingFilesJob, + IIntegrityOrphanedFilesJob, + IIntegrityPathWithChecksumJob, + IIntegrityPathWithReportJob, +} from 'src/types'; +import { ImmichFileResponse } from 'src/utils/file'; +import { handlePromiseError } from 'src/utils/misc'; + +/** + * Orphan Files: + * Files are detected in /data/encoded-video, /data/library, /data/upload + * Checked against the asset table + * Files are detected in /data/thumbs + * Checked against the asset_file table + * + * * Can perform download or delete of files + * + * Missing Files: + * Paths are queried from asset(originalPath, encodedVideoPath), asset_file(path) + * Check whether files exist on disk + * + * * Reports must include origin (asset or asset_file) & ID for further action + * * Can perform trash (asset) or delete (asset_file) + * + * Checksum Mismatch: + * Paths & checksums are queried from asset(originalPath, checksum) + * Check whether files match checksum, missing files ignored + * + * * Reports must include origin (as above) for further action + * * Can perform download or trash (asset) + */ + +@Injectable() +export class IntegrityService extends BaseService { + private integrityLock = false; + + @OnEvent({ name: 'ConfigInit', workers: [ImmichWorker.Microservices] }) + async onConfigInit({ + newConfig: { + integrityChecks: { orphanedFiles, missingFiles, checksumFiles }, + }, + }: ArgOf<'ConfigInit'>) { + this.integrityLock = await this.databaseRepository.tryLock(DatabaseLock.IntegrityCheck); + if (this.integrityLock) { + this.cronRepository.create({ + name: 'integrityOrphanedFiles', + expression: orphanedFiles.cronExpression, + onTick: () => + handlePromiseError( + this.jobRepository.queue({ name: JobName.IntegrityOrphanedFilesQueueAll, data: {} }), + this.logger, + ), + start: orphanedFiles.enabled, + }); + + this.cronRepository.create({ + name: 'integrityMissingFiles', + expression: missingFiles.cronExpression, + onTick: () => + handlePromiseError( + this.jobRepository.queue({ name: JobName.IntegrityMissingFilesQueueAll, data: {} }), + this.logger, + ), + start: missingFiles.enabled, + }); + + this.cronRepository.create({ + name: 'integrityChecksumFiles', + expression: checksumFiles.cronExpression, + onTick: () => + handlePromiseError(this.jobRepository.queue({ name: JobName.IntegrityChecksumFiles, data: {} }), this.logger), + start: checksumFiles.enabled, + }); + } + + // debug: run on boot + setTimeout(() => { + void this.jobRepository.queue({ + name: JobName.IntegrityOrphanedFilesQueueAll, + data: {}, + }); + + void this.jobRepository.queue({ + name: JobName.IntegrityMissingFilesQueueAll, + data: {}, + }); + + void this.jobRepository.queue({ + name: JobName.IntegrityChecksumFiles, + data: {}, + }); + }, 1000); + } + + @OnEvent({ name: 'ConfigUpdate', server: true }) + onConfigUpdate({ + newConfig: { + integrityChecks: { orphanedFiles, missingFiles, checksumFiles }, + }, + }: ArgOf<'ConfigUpdate'>) { + if (!this.integrityLock) { + return; + } + + this.cronRepository.update({ + name: 'integrityOrphanedFiles', + expression: orphanedFiles.cronExpression, + start: orphanedFiles.enabled, + }); + + this.cronRepository.update({ + name: 'integrityMissingFiles', + expression: missingFiles.cronExpression, + start: missingFiles.enabled, + }); + + this.cronRepository.update({ + name: 'integrityChecksumFiles', + expression: checksumFiles.cronExpression, + start: checksumFiles.enabled, + }); + } + + getIntegrityReportSummary(): Promise { + return this.integrityRepository.getIntegrityReportSummary(); + } + + async getIntegrityReport(dto: IntegrityGetReportDto): Promise { + return { + items: await this.integrityRepository.getIntegrityReports(dto.type), + }; + } + + getIntegrityReportCsv(type: IntegrityReportType): Readable { + return this.integrityRepository.streamIntegrityReportsCSV(type); + } + + async getIntegrityReportFile(id: string): Promise { + const { path } = await this.integrityRepository.getById(id); + + return new ImmichFileResponse({ + path, + fileName: basename(path), + contentType: 'application/octet-stream', + cacheControl: CacheControl.PrivateWithoutCache, + }); + } + + async deleteIntegrityReport(auth: AuthDto, id: string): Promise { + const { path, assetId, fileAssetId } = await this.integrityRepository.getById(id); + + if (assetId) { + await this.assetRepository.updateAll([assetId], { + deletedAt: new Date(), + status: AssetStatus.Trashed, + }); + + await this.eventRepository.emit('AssetTrashAll', { + assetIds: [assetId], + userId: auth.user.id, + }); + + await this.integrityRepository.deleteById(id); + } else if (fileAssetId) { + await this.assetRepository.deleteFiles([{ id: fileAssetId }]); + } else { + await this.storageRepository.unlink(path); + await this.integrityRepository.deleteById(id); + } + } + + @OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.IntegrityCheck }) + async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise { + this.logger.log(`Checking for out of date orphaned file reports...`); + + const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.OrphanFile); + + let total = 0; + for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityOrphanedFilesRefresh, + data: { + items: batchReports, + }, + }); + + total += batchReports.length; + this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`); + } + + if (refreshOnly) { + this.logger.log('Refresh complete.'); + return JobStatus.Success; + } + + this.logger.log(`Scanning for orphaned files...`); + + const assetPaths = this.storageRepository.walk({ + pathsToCrawl: [StorageFolder.EncodedVideo, StorageFolder.Library, StorageFolder.Upload].map((folder) => + StorageCore.getBaseFolder(folder), + ), + includeHidden: false, + take: JOBS_LIBRARY_PAGINATION_SIZE, + }); + + const assetFilePaths = this.storageRepository.walk({ + pathsToCrawl: [StorageCore.getBaseFolder(StorageFolder.Thumbnails)], + includeHidden: false, + take: JOBS_LIBRARY_PAGINATION_SIZE, + }); + + async function* paths() { + for await (const batch of assetPaths) { + yield ['asset', batch] as const; + } + + for await (const batch of assetFilePaths) { + yield ['asset_file', batch] as const; + } + } + + total = 0; + for await (const [batchType, batchPaths] of paths()) { + await this.jobRepository.queue({ + name: JobName.IntegrityOrphanedFiles, + data: { + type: batchType, + paths: batchPaths, + }, + }); + + const count = batchPaths.length; + total += count; + + this.logger.log(`Queued orphan check of ${count} file(s) (${total} so far)`); + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityOrphanedFiles, queue: QueueName.IntegrityCheck }) + async handleOrphanedFiles({ type, paths }: IIntegrityOrphanedFilesJob): Promise { + this.logger.log(`Processing batch of ${paths.length} files to check if they are orphaned.`); + + const orphanedFiles = new Set(paths); + if (type === 'asset') { + const assets = await this.integrityRepository.getAssetPathsByPaths(paths); + for (const { originalPath, encodedVideoPath } of assets) { + orphanedFiles.delete(originalPath); + + if (encodedVideoPath) { + orphanedFiles.delete(encodedVideoPath); + } + } + } else { + const assets = await this.integrityRepository.getAssetFilePathsByPaths(paths); + for (const { path } of assets) { + orphanedFiles.delete(path); + } + } + + if (orphanedFiles.size > 0) { + await this.integrityRepository.create( + [...orphanedFiles].map((path) => ({ + type: IntegrityReportType.OrphanFile, + path, + })), + ); + } + + this.logger.log(`Processed ${paths.length} and found ${orphanedFiles.size} orphaned file(s).`); + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityOrphanedFilesRefresh, queue: QueueName.IntegrityCheck }) + async handleOrphanedRefresh({ items }: IIntegrityPathWithReportJob): Promise { + this.logger.log(`Processing batch of ${items.length} reports to check if they are out of date.`); + + const results = await Promise.all( + items.map(({ reportId, path }) => + stat(path) + .then(() => void 0) + .catch(() => reportId), + ), + ); + + const reportIds = results.filter(Boolean) as string[]; + + if (reportIds.length > 0) { + await this.integrityRepository.deleteByIds(reportIds); + } + + this.logger.log(`Processed ${items.length} paths and found ${reportIds.length} report(s) out of date.`); + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.IntegrityCheck }) + async handleMissingFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise { + if (refreshOnly) { + this.logger.log(`Checking for out of date missing file reports...`); + + const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.MissingFile); + + let total = 0; + for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityMissingFilesRefresh, + data: { + items: batchReports, + }, + }); + + total += batchReports.length; + this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`); + } + + this.logger.log('Refresh complete.'); + return JobStatus.Success; + } + + this.logger.log(`Scanning for missing files...`); + + const assetPaths = this.integrityRepository.streamAssetPaths(); + + let total = 0; + for await (const batchPaths of chunk(assetPaths, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityMissingFiles, + data: { + items: batchPaths, + }, + }); + + total += batchPaths.length; + this.logger.log(`Queued missing check of ${batchPaths.length} file(s) (${total} so far)`); + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityMissingFiles, queue: QueueName.IntegrityCheck }) + async handleMissingFiles({ items }: IIntegrityMissingFilesJob): Promise { + this.logger.log(`Processing batch of ${items.length} files to check if they are missing.`); + + const results = await Promise.all( + items.map((item) => + stat(item.path) + .then(() => ({ ...item, exists: true })) + .catch(() => ({ ...item, exists: false })), + ), + ); + + const outdatedReports = results + .filter(({ exists, reportId }) => exists && reportId) + .map(({ reportId }) => reportId!); + + if (outdatedReports.length > 0) { + await this.integrityRepository.deleteByIds(outdatedReports); + } + + const missingFiles = results.filter(({ exists }) => !exists); + if (missingFiles.length > 0) { + await this.integrityRepository.create( + missingFiles.map(({ path, assetId, fileAssetId }) => ({ + type: IntegrityReportType.MissingFile, + path, + assetId, + fileAssetId, + })), + ); + } + + this.logger.log(`Processed ${items.length} and found ${missingFiles.length} missing file(s).`); + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityMissingFilesRefresh, queue: QueueName.IntegrityCheck }) + async handleMissingRefresh({ items: paths }: IIntegrityPathWithReportJob): Promise { + this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`); + + const results = await Promise.all( + paths.map(({ reportId, path }) => + stat(path) + .then(() => reportId) + .catch(() => void 0), + ), + ); + + const reportIds = results.filter(Boolean) as string[]; + + if (reportIds.length > 0) { + await this.integrityRepository.deleteByIds(reportIds); + } + + this.logger.log(`Processed ${paths.length} paths and found ${reportIds.length} report(s) out of date.`); + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.IntegrityCheck }) + async handleChecksumFiles({ refreshOnly }: IIntegrityJob = {}): Promise { + if (refreshOnly) { + this.logger.log(`Checking for out of date checksum file reports...`); + + const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.ChecksumFail); + + let total = 0; + for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityChecksumFilesRefresh, + data: { + items: batchReports.map(({ path, reportId, checksum }) => ({ + path, + reportId, + checksum: checksum?.toString('hex'), + })), + }, + }); + + total += batchReports.length; + this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`); + } + + this.logger.log('Refresh complete.'); + return JobStatus.Success; + } + + const { + integrityChecks: { + checksumFiles: { timeLimit, percentageLimit }, + }, + } = await this.getConfig({ + withCache: true, + }); + + this.logger.log( + `Checking file checksums... (will run for up to ${(timeLimit / (60 * 60 * 1000)).toFixed(2)} hours or until ${(percentageLimit * 100).toFixed(2)}% of assets are processed)`, + ); + + let processed = 0; + const startedAt = Date.now(); + const { count } = await this.integrityRepository.getAssetCount(); + const checkpoint = await this.systemMetadataRepository.get(SystemMetadataKey.IntegrityChecksumCheckpoint); + + let startMarker: Date | undefined = checkpoint?.date ? new Date(checkpoint.date) : undefined; + let endMarker: Date | undefined; + + const printStats = () => { + const averageTime = ((Date.now() - startedAt) / processed).toFixed(2); + const completionProgress = ((processed / count) * 100).toFixed(2); + + this.logger.log( + `Processed ${processed} files so far... (avg. ${averageTime} ms/asset, ${completionProgress}% of all assets)`, + ); + }; + + let lastCreatedAt: Date | undefined; + + finishEarly: do { + this.logger.log( + `Processing assets in range [${startMarker?.toISOString() ?? 'beginning'}, ${endMarker?.toISOString() ?? 'end'}]`, + ); + + const assets = this.integrityRepository.streamAssetChecksums(startMarker, endMarker); + endMarker = startMarker; + startMarker = undefined; + + for await (const { originalPath, checksum, createdAt, assetId, reportId } of assets) { + processed++; + + try { + const hash = createHash('sha1'); + + await pipeline([ + createReadStream(originalPath), + new Writable({ + write(chunk, _encoding, callback) { + hash.update(chunk); + callback(); + }, + }), + ]); + + if (checksum.equals(hash.digest())) { + if (reportId) { + await this.integrityRepository.deleteById(reportId); + } + } else { + throw new Error('File failed checksum'); + } + } catch (error) { + if ((error as { code?: string }).code === 'ENOENT') { + if (reportId) { + await this.integrityRepository.deleteById(reportId); + } + // missing file; handled by the missing files job + continue; + } + + this.logger.warn('Failed to process a file: ' + error); + await this.integrityRepository.create({ + path: originalPath, + type: IntegrityReportType.ChecksumFail, + assetId, + }); + } + + if (processed % 100 === 0) { + printStats(); + } + + if (Date.now() > startedAt + timeLimit || processed > count * percentageLimit) { + this.logger.log('Reached stop criteria.'); + lastCreatedAt = createdAt; + break finishEarly; + } + } + } while (endMarker); + + await this.systemMetadataRepository.set(SystemMetadataKey.IntegrityChecksumCheckpoint, { + date: lastCreatedAt?.toISOString(), + }); + + printStats(); + + if (lastCreatedAt) { + this.logger.log(`Finished checksum job, will continue from ${lastCreatedAt.toISOString()}.`); + } else { + this.logger.log(`Finished checksum job, covered all assets.`); + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityChecksumFilesRefresh, queue: QueueName.IntegrityCheck }) + async handleChecksumRefresh({ items: paths }: IIntegrityPathWithChecksumJob): Promise { + this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`); + + const results = await Promise.all( + paths.map(async ({ reportId, path, checksum }) => { + if (!checksum) { + return reportId; + } + + try { + const hash = createHash('sha1'); + + await pipeline([ + createReadStream(path), + new Writable({ + write(chunk, _encoding, callback) { + hash.update(chunk); + callback(); + }, + }), + ]); + + if (Buffer.from(checksum, 'hex').equals(hash.digest())) { + return reportId; + } + } catch (error) { + if ((error as { code?: string }).code === 'ENOENT') { + return reportId; + } + } + }), + ); + + const reportIds = results.filter(Boolean) as string[]; + + if (reportIds.length > 0) { + await this.integrityRepository.deleteByIds(reportIds); + } + + this.logger.log(`Processed ${paths.length} paths and found ${reportIds.length} report(s) out of date.`); + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityReportDelete, queue: QueueName.IntegrityCheck }) + async handleDeleteIntegrityReport({ type }: IIntegrityDeleteReportJob): Promise { + this.logger.log(`Deleting all entries for ${type ?? 'all types of'} integrity report`); + + let properties; + switch (type) { + case IntegrityReportType.ChecksumFail: { + properties = ['assetId'] as const; + break; + } + case IntegrityReportType.MissingFile: { + properties = ['assetId', 'fileAssetId'] as const; + break; + } + case IntegrityReportType.OrphanFile: { + properties = [void 0] as const; + break; + } + default: { + properties = [void 0, 'assetId', 'fileAssetId'] as const; + break; + } + } + + for (const property of properties) { + const reports = this.integrityRepository.streamIntegrityReportsByProperty(property, type); + for await (const report of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + // todo: queue sub-job here instead? + + switch (property) { + case 'assetId': { + const ids = report.map(({ assetId }) => assetId!); + await this.assetRepository.updateAll(ids, { + deletedAt: new Date(), + status: AssetStatus.Trashed, + }); + + await this.eventRepository.emit('AssetTrashAll', { + assetIds: ids, + userId: '', // ??? + }); + + await this.integrityRepository.deleteByIds(report.map(({ id }) => id)); + break; + } + case 'fileAssetId': { + await this.assetRepository.deleteFiles(report.map(({ fileAssetId }) => ({ id: fileAssetId! }))); + break; + } + default: { + await Promise.all(report.map(({ path }) => this.storageRepository.unlink(path).catch(() => void 0))); + await this.integrityRepository.deleteByIds(report.map(({ id }) => id)); + break; + } + } + } + } + + this.logger.log('Finished deleting integrity report.'); + return JobStatus.Success; + } +} + +async function* chunk(generator: AsyncIterableIterator, n: number) { + let chunk: T[] = []; + for await (const item of generator) { + chunk.push(item); + + if (chunk.length === n) { + yield chunk; + chunk = []; + } + } + + if (chunk.length > 0) { + yield chunk; + } +} diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index b57a203788..d0b41572b6 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -2,7 +2,7 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { OnEvent } from 'src/decorators'; import { mapAsset } from 'src/dtos/asset-response.dto'; import { JobCreateDto } from 'src/dtos/job.dto'; -import { AssetType, AssetVisibility, JobName, JobStatus, ManualJobName } from 'src/enum'; +import { AssetType, AssetVisibility, IntegrityReportType, JobName, JobStatus, ManualJobName } from 'src/enum'; import { ArgsOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; import { JobItem } from 'src/types'; @@ -34,6 +34,42 @@ const asJobItem = (dto: JobCreateDto): JobItem => { return { name: JobName.DatabaseBackup }; } + case ManualJobName.IntegrityMissingFiles: { + return { name: JobName.IntegrityMissingFilesQueueAll }; + } + + case ManualJobName.IntegrityOrphanFiles: { + return { name: JobName.IntegrityOrphanedFilesQueueAll }; + } + + case ManualJobName.IntegrityChecksumFiles: { + return { name: JobName.IntegrityChecksumFiles }; + } + + case ManualJobName.IntegrityMissingFilesRefresh: { + return { name: JobName.IntegrityMissingFilesQueueAll, data: { refreshOnly: true } }; + } + + case ManualJobName.IntegrityOrphanFilesRefresh: { + return { name: JobName.IntegrityOrphanedFilesQueueAll, data: { refreshOnly: true } }; + } + + case ManualJobName.IntegrityChecksumFilesRefresh: { + return { name: JobName.IntegrityChecksumFiles, data: { refreshOnly: true } }; + } + + case ManualJobName.IntegrityMissingFilesDeleteAll: { + return { name: JobName.IntegrityReportDelete, data: { type: IntegrityReportType.MissingFile } }; + } + + case ManualJobName.IntegrityOrphanFilesDeleteAll: { + return { name: JobName.IntegrityReportDelete, data: { type: IntegrityReportType.OrphanFile } }; + } + + case ManualJobName.IntegrityChecksumFilesDeleteAll: { + return { name: JobName.IntegrityReportDelete, data: { type: IntegrityReportType.ChecksumFail } }; + } + default: { throw new BadRequestException('Invalid job name'); } diff --git a/server/src/services/queue.service.spec.ts b/server/src/services/queue.service.spec.ts index f5cf20413e..a11a265796 100644 --- a/server/src/services/queue.service.spec.ts +++ b/server/src/services/queue.service.spec.ts @@ -23,7 +23,7 @@ describe(QueueService.name, () => { it('should update concurrency', () => { sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig }); - expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(17); + expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(18); expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(5, QueueName.FacialRecognition, 1); expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(7, QueueName.DuplicateDetection, 1); expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(8, QueueName.BackgroundTask, 5); @@ -77,6 +77,7 @@ describe(QueueService.name, () => { [QueueName.BackupDatabase]: expected, [QueueName.Ocr]: expected, [QueueName.Workflow]: expected, + [QueueName.IntegrityCheck]: expected, }); }); }); diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index fbdd655bbc..f6ba209951 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -41,6 +41,7 @@ const updatedConfig = Object.freeze({ [QueueName.Notification]: { concurrency: 5 }, [QueueName.Ocr]: { concurrency: 1 }, [QueueName.Workflow]: { concurrency: 5 }, + [QueueName.IntegrityCheck]: { concurrency: 1 }, }, backup: { database: { @@ -72,6 +73,22 @@ const updatedConfig = Object.freeze({ accelDecode: false, tonemap: ToneMapping.Hable, }, + integrityChecks: { + orphanedFiles: { + enabled: true, + cronExpression: '0 03 * * *', + }, + missingFiles: { + enabled: true, + cronExpression: '0 03 * * *', + }, + checksumFiles: { + enabled: true, + cronExpression: '0 03 * * *', + timeLimit: 60 * 60 * 1000, + percentageLimit: 1, + }, + }, logging: { enabled: true, level: LogLevel.Log, diff --git a/server/src/types.ts b/server/src/types.ts index e404332fac..3a5c9acacc 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -9,6 +9,7 @@ import { DatabaseSslMode, ExifOrientation, ImageFormat, + IntegrityReportType, JobName, MemoryType, PluginTriggerType, @@ -276,6 +277,36 @@ export interface IWorkflowJob { event: WorkflowData[T]; } +export interface IIntegrityJob { + refreshOnly?: boolean; +} + +export interface IIntegrityDeleteReportJob { + type?: IntegrityReportType; +} + +export interface IIntegrityOrphanedFilesJob { + type: 'asset' | 'asset_file'; + paths: string[]; +} + +export interface IIntegrityMissingFilesJob { + items: { + path: string; + reportId: string | null; + assetId: string | null; + fileAssetId: string | null; + }[]; +} + +export interface IIntegrityPathWithReportJob { + items: { path: string; reportId: string | null }[]; +} + +export interface IIntegrityPathWithChecksumJob { + items: { path: string; reportId: string | null; checksum?: string | null }[]; +} + export interface JobCounts { active: number; completed: number; @@ -385,7 +416,18 @@ export type JobItem = | { name: JobName.Ocr; data: IEntityJob } // Workflow - | { name: JobName.WorkflowRun; data: IWorkflowJob }; + | { name: JobName.WorkflowRun; data: IWorkflowJob } + + // Integrity + | { name: JobName.IntegrityOrphanedFilesQueueAll; data?: IIntegrityJob } + | { name: JobName.IntegrityOrphanedFiles; data: IIntegrityOrphanedFilesJob } + | { name: JobName.IntegrityOrphanedFilesRefresh; data: IIntegrityPathWithReportJob } + | { name: JobName.IntegrityMissingFilesQueueAll; data?: IIntegrityJob } + | { name: JobName.IntegrityMissingFiles; data: IIntegrityPathWithReportJob } + | { name: JobName.IntegrityMissingFilesRefresh; data: IIntegrityPathWithReportJob } + | { name: JobName.IntegrityChecksumFiles; data?: IIntegrityJob } + | { name: JobName.IntegrityChecksumFilesRefresh; data?: IIntegrityPathWithChecksumJob } + | { name: JobName.IntegrityReportDelete; data: IIntegrityDeleteReportJob }; export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number]; @@ -500,6 +542,7 @@ export interface SystemMetadata extends Record; [SystemMetadataKey.VersionCheckState]: VersionCheckMetadata; [SystemMetadataKey.MemoriesState]: MemoriesState; + [SystemMetadataKey.IntegrityChecksumCheckpoint]: { date?: string }; } export interface UserPreferences { diff --git a/server/src/validation.ts b/server/src/validation.ts index 6d4bbfbe36..da5033dc32 100644 --- a/server/src/validation.ts +++ b/server/src/validation.ts @@ -33,6 +33,7 @@ import { import { CronJob } from 'cron'; import { DateTime } from 'luxon'; import sanitize from 'sanitize-filename'; +import { IntegrityReportType } from 'src/enum'; import { isIP, isIPRange } from 'validator'; @Injectable() @@ -96,6 +97,12 @@ export class UUIDAssetIDParamDto { assetId!: string; } +export class IntegrityReportTypeParamDto { + @IsNotEmpty() + @ApiProperty({ enum: IntegrityReportType, enumName: 'IntegrityReportType' }) + type!: IntegrityReportType; +} + type PinCodeOptions = { optional?: boolean } & OptionalOptions; export const PinCode = (options?: PinCodeOptions & ApiPropertyOptions) => { const { optional, nullable, emptyToNull, ...apiPropertyOptions } = { diff --git a/server/test/utils.ts b/server/test/utils.ts index 77853f897a..a274bde0fb 100644 --- a/server/test/utils.ts +++ b/server/test/utils.ts @@ -31,6 +31,7 @@ import { DownloadRepository } from 'src/repositories/download.repository'; import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityRepository } from 'src/repositories/integrity.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -225,6 +226,7 @@ export type ServiceOverrides = { duplicateRepository: DuplicateRepository; email: EmailRepository; event: EventRepository; + integrityReport: IntegrityRepository; job: JobRepository; library: LibraryRepository; logger: LoggingRepository; @@ -298,6 +300,7 @@ export const getMocks = () => { email: automock(EmailRepository, { args: [loggerMock] }), // eslint-disable-next-line no-sparse-arrays event: automock(EventRepository, { args: [, , loggerMock], strict: false }), + integrityReport: automock(IntegrityRepository, { strict: false }), job: newJobRepositoryMock(), apiKey: automock(ApiKeyRepository), library: automock(LibraryRepository, { strict: false }), @@ -366,6 +369,7 @@ export const newTestService = ( overrides.duplicateRepository || (mocks.duplicateRepository as As), overrides.email || (mocks.email as As), overrides.event || (mocks.event as As), + overrides.integrityReport || (mocks.integrityReport as As), overrides.job || (mocks.job as As), overrides.library || (mocks.library as As), overrides.machineLearning || (mocks.machineLearning as As), diff --git a/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte b/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte index 61d4c643c8..f48d48805d 100644 --- a/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte +++ b/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte @@ -1,15 +1,17 @@ @@ -12,6 +12,7 @@ + diff --git a/web/src/lib/utils.ts b/web/src/lib/utils.ts index 100f807273..6b6ceddd64 100644 --- a/web/src/lib/utils.ts +++ b/web/src/lib/utils.ts @@ -163,6 +163,7 @@ export const getQueueName = derived(t, ($t) => { [QueueName.BackupDatabase]: $t('admin.backup_database'), [QueueName.Ocr]: $t('admin.machine_learning_ocr'), [QueueName.Workflow]: $t('workflow'), + [QueueName.IntegrityCheck]: 'TODO', }; return names[name]; diff --git a/web/src/routes/admin/maintenance/+page.svelte b/web/src/routes/admin/maintenance/+page.svelte new file mode 100644 index 0000000000..7021d8a558 --- /dev/null +++ b/web/src/routes/admin/maintenance/+page.svelte @@ -0,0 +1,149 @@ + + + +
+
+

{$t('admin.maintenance_integrity_report')}

+ + +
+
+ diff --git a/web/src/routes/admin/maintenance/+page.ts b/web/src/routes/admin/maintenance/+page.ts new file mode 100644 index 0000000000..f514d6e32d --- /dev/null +++ b/web/src/routes/admin/maintenance/+page.ts @@ -0,0 +1,17 @@ +import { authenticate } from '$lib/utils/auth'; +import { getFormatter } from '$lib/utils/i18n'; +import { getIntegrityReportSummary } from '@immich/sdk'; +import type { PageLoad } from './$types'; + +export const load = (async ({ url }) => { + await authenticate(url, { admin: true }); + const integrityReport = await getIntegrityReportSummary(); + const $t = await getFormatter(); + + return { + integrityReport, + meta: { + title: $t('admin.maintenance_settings'), + }, + }; +}) satisfies PageLoad; diff --git a/web/src/routes/admin/maintenance/integrity-report/[type]/+page.svelte b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.svelte new file mode 100644 index 0000000000..182cc61323 --- /dev/null +++ b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.svelte @@ -0,0 +1,171 @@ + + + { + location.href = `${getBaseUrl()}/admin/maintenance/integrity/report/${data.type}/csv`; + }, + }, + { + title: 'Delete All', + onAction: removeAll, + icon: mdiTrashCanOutline, + }, + ]} +> +
+
+ + + + + + + + + {#each integrityReport as { id, path } (id)} + + + + + {/each} + +
{$t('filename')}
{path} + handleOpen(event, { position: 'top-right' }, id)} + aria-label={$t('open')} + disabled={deleting.has(id) || deleting.has('all')} + />
+
+
+ diff --git a/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts new file mode 100644 index 0000000000..306cc1ac56 --- /dev/null +++ b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts @@ -0,0 +1,24 @@ +import { authenticate } from '$lib/utils/auth'; +import { getFormatter } from '$lib/utils/i18n'; +import { getIntegrityReport, IntegrityReportType } from '@immich/sdk'; +import type { PageLoad } from './$types'; + +export const load = (async ({ params, url }) => { + const type = params.type as IntegrityReportType; + + await authenticate(url, { admin: true }); + const integrityReport = await getIntegrityReport({ + integrityGetReportDto: { + type, + }, + }); + const $t = await getFormatter(); + + return { + type, + integrityReport, + meta: { + title: $t(`admin.maintenance_integrity_${type}`), + }, + }; +}) satisfies PageLoad;