a/immich
1
0
Fork 0
mirror of https://github.com/Art051/immich.git synced 2025-08-11 19:29:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(server): Enqueue jobs in bulk (#5974)

Browse Source 
* feat(server): Enqueue jobs in bulk

The Job Repository now has a `queueAll` method, that enqueues messages
in bulk (using BullMQ's
[`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)),
improving performance when many jobs must be enqueued within the same
operation.

Primary change is in `src/domain/job/job.service.ts`, and other services
have been refactored to use `queueAll` when useful.

As a simple local benchmark, triggering a full thumbnail generation
process over a library of ~1,200 assets and ~350 faces went from
**~600ms** to **~250ms**.

* fix: Review feedback
This commit is contained in:
Michael Manganiello
2024-01-01 15:45:42 -05:00
committed by GitHub
parent 7dd88c4114
commit 4a5b8c3770
20 changed files with 323 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
server/src/domain/media/media.service.ts
View File

@@ -21,6 +21,7 @@ import {
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
JobItem,
VideoCodecHWConfig,
VideoStreamInfo,
WithoutProperty,
@@ -74,22 +75,27 @@ export class MediaService {
});
for await (const assets of assetPagination) {
const jobs: JobItem[] = [];
for (const asset of assets) {
if (!asset.resizePath || force) {
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } });
jobs.push({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } });
continue;
}
if (!asset.webpPath) {
await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { id: asset.id } });
jobs.push({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { id: asset.id } });
}
if (!asset.thumbhash) {
await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: { id: asset.id } });
jobs.push({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: { id: asset.id } });
}
}
await this.jobRepository.queueAll(jobs);
}
const people = force ? await this.personRepository.getAll() : await this.personRepository.getAllWithoutThumbnail();
const jobs: JobItem[] = [];
for (const person of people) {
if (!person.faceAssetId) {
const face = await this.personRepository.getRandomFace(person.id);
@@ -100,9 +106,11 @@ export class MediaService {
await this.personRepository.update({ id: person.id, faceAssetId: face.assetId });
}
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } });
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } });
}
await this.jobRepository.queueAll(jobs);
return true;
}
@@ -118,15 +126,15 @@ export class MediaService {
}
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.MIGRATE_ASSET, data: { id: asset.id } });
}
await this.jobRepository.queueAll(
assets.map((asset) => ({ name: JobName.MIGRATE_ASSET, data: { id: asset.id } })),
);
}
const people = await this.personRepository.getAll();
for (const person of people) {
await this.jobRepository.queue({ name: JobName.MIGRATE_PERSON, data: { id: person.id } });
}
await this.jobRepository.queueAll(
people.map((person) => ({ name: JobName.MIGRATE_PERSON, data: { id: person.id } })),
);
return true;
}
@@ -224,9 +232,9 @@ export class MediaService {
});
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { id: asset.id } });
}
await this.jobRepository.queueAll(
assets.map((asset) => ({ name: JobName.VIDEO_CONVERSION, data: { id: asset.id } })),
);
}
return true;