import * as fastq from 'fastq'; import { uniqueId } from 'lodash-es'; export type Task = { readonly id: string; status: 'idle' | 'processing' | 'succeeded' | 'failed'; data: T; error: unknown | undefined; count: number; // TODO: Could be useful to adding progress property. // TODO: Could be useful to adding start_at/end_at/duration properties. result: undefined | R; }; export type QueueOptions = { verbose?: boolean; concurrency?: number; retry?: number; // TODO: Could be useful to adding timeout property for retry. }; export type ComputedQueueOptions = Required; export const defaultQueueOptions = { concurrency: 1, retry: 0, verbose: false, }; /** * An in-memory queue that processes tasks in parallel with a given concurrency. * @see {@link https://www.npmjs.com/package/fastq} * @template T - The type of the worker task data. * @template R - The type of the worker output data. */ export class Queue { private readonly queue: fastq.queueAsPromised>; private readonly store = new Map>(); readonly options: ComputedQueueOptions; readonly worker: (data: T) => Promise; /** * Create a new queue. * @param worker - The worker function that processes the task. * @param options - The queue options. */ constructor(worker: (data: T) => Promise, options?: QueueOptions) { this.options = { ...defaultQueueOptions, ...options }; this.worker = worker; this.store = new Map>(); this.queue = this.buildQueue(); } get tasks(): Task[] { const tasks: Task[] = []; for (const task of this.store.values()) { tasks.push(task); } return tasks; } getTask(id: string): Task { const task = this.store.get(id); if (!task) { throw new Error(`Task with id ${id} not found`); } return task; } /** * Wait for the queue to be empty. * @returns Promise - The returned Promise will be resolved when all tasks in the queue have been processed by a worker. * This promise could be ignored as it will not lead to a `unhandledRejection`. */ async drained(): Promise { await this.queue.drain(); } /** * Add a task at the end of the queue. * @see {@link https://www.npmjs.com/package/fastq} * @param data * @returns Promise - A Promise that will be fulfilled (rejected) when the task is completed successfully (unsuccessfully). * This promise could be ignored as it will not lead to a `unhandledRejection`. */ async push(data: T): Promise> { const id = uniqueId(); const task: Task = { id, status: 'idle', error: undefined, count: 0, data, result: undefined }; this.store.set(id, task); return this.queue.push(id); } // TODO: Support more function delegation to fastq. private buildQueue(): fastq.queueAsPromised> { return fastq.promise((id: string) => { const task = this.getTask(id); return this.work(task); }, this.options.concurrency); } private async work(task: Task): Promise> { task.count += 1; task.error = undefined; task.status = 'processing'; if (this.options.verbose) { console.log('[task] processing:', task); } try { task.result = await this.worker(task.data); task.status = 'succeeded'; if (this.options.verbose) { console.log('[task] succeeded:', task); } return task; } catch (error) { task.error = error; task.status = 'failed'; if (this.options.verbose) { console.log('[task] failed:', task); } if (this.options.retry > 0 && task.count < this.options.retry) { if (this.options.verbose) { console.log('[task] retry:', task); } return this.work(task); } return task; } } }