Background jobs are essential when a Node.js application needs to perform work without making the user wait for the result. Sending emails, processing reports, resizing images, generating invoices, syncing external APIs, and cleaning temporary data are all good candidates for asynchronous processing.
The key idea is simple: move expensive or non-critical work away from the request-response cycle. In this article, we will build a small in-memory background job queue using only Node.js, while covering job states, retries, concurrency, failures, and graceful shutdown.
Building a Reliable Background Job Queue
A background job system usually has three important parts: a producer that creates jobs, a queue that stores pending work, and a worker that processes jobs. In production, the queue is commonly backed by Redis, a database, or a dedicated message broker so jobs survive application restarts.
For this example, we will keep the implementation intentionally small and dependency-free. The queue will support configurable concurrency, job status tracking, automatic retries, and graceful shutdown, making it useful for understanding the architecture before introducing tools such as BullMQ or RabbitMQ.
A job should contain enough information for the worker to execute it independently from the original request. The worker should also treat failures as expected events rather than crashing the entire process. Retry limits are especially important because repeatedly retrying a permanently invalid job can create unnecessary load and prevent healthy jobs from progressing.
The example below simulates email processing with random failures. Run it with a recent Node.js version using node background-jobs.js, then observe how jobs move through queued, processing, completed, and failed states while the worker controls how many jobs execute concurrently.
const { randomUUID } = require('node:crypto');
// Simple in-memory queue for demonstrating background job concepts.
class JobQueue {
constructor({ concurrency = 2, maxRetries = 2 } = {}) {
this.jobs = [];
this.activeJobs = new Set();
this.concurrency = concurrency;
this.maxRetries = maxRetries;
this.isRunning = false;
this.isShuttingDown = false;
}
add(type, payload) {
const job = {
id: randomUUID(),
type,
payload,
status: 'queued',
attempts: 0,
createdAt: new Date().toISOString()
};
this.jobs.push(job);
console.log(`[QUEUE] Added job ${job.id.slice(0, 8)} (${job.type})`);
return job;
}
getNextJob() {
return this.jobs.find((job) => job.status === 'queued');
}
async start() {
if (this.isRunning) return;
this.isRunning = true;
console.log(`[WORKER] Started with concurrency: ${this.concurrency}`);
while (!this.isShuttingDown) {
while (this.activeJobs.size < this.concurrency) {
const job = this.getNextJob();
if (!job) break;
job.status = 'processing';
job.startedAt = new Date().toISOString();
this.activeJobs.add(job.id);
this.process(job).finally(() => {
this.activeJobs.delete(job.id);
});
}
await this.sleep(100);
}
console.log('[WORKER] Shutdown signal received. Waiting for active jobs...');
while (this.activeJobs.size > 0) {
await this.sleep(100);
}
this.isRunning = false;
console.log('[WORKER] All active jobs finished. Worker stopped.');
}
async process(job) {
job.attempts += 1;
console.log(`[WORKER] Processing ${job.id.slice(0, 8)} - attempt ${job.attempts}`);
try {
await this.execute(job);
job.status = 'completed';
job.completedAt = new Date().toISOString();
console.log(`[WORKER] Completed ${job.id.slice(0, 8)}`);
} catch (error) {
console.error(`[WORKER] Failed ${job.id.slice(0, 8)}: ${error.message}`);
if (job.attempts <= this.maxRetries) {
job.status = 'queued';
console.log(`[WORKER] Retrying ${job.id.slice(0, 8)}...`);
} else {
job.status = 'failed';
job.failedAt = new Date().toISOString();
job.error = error.message;
console.error(`[WORKER] Permanently failed ${job.id.slice(0, 8)}`);
}
}
}
async execute(job) {
if (job.type === 'send-email') {
console.log(`[JOB] Sending email to ${job.payload.email}`);
await this.sleep(500 + Math.random() * 1000);
// Simulate an unreliable external email provider.
if (Math.random() < 0.25) {
throw new Error('Email provider temporarily unavailable');
}
console.log(`[JOB] Email sent to ${job.payload.email}`);
return;
}
throw new Error(`Unknown job type: ${job.type}`);
}
sleep(milliseconds) {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
shutdown() {
console.log('[WORKER] Graceful shutdown requested...');
this.isShuttingDown = true;
}
printSummary() {
const summary = this.jobs.reduce((result, job) => {
result[job.status] = (result[job.status] || 0) + 1;
return result;
}, {});
console.log('\n[SUMMARY] Job statistics:');
console.table(summary);
}
}
async function main() {
console.log('[APP] Creating background job queue...');
const queue = new JobQueue({
concurrency: 3,
maxRetries: 2
});
// Start the worker without blocking job creation.
const workerPromise = queue.start();
const users = [
'alice@example.com',
'bob@example.com',
'charlie@example.com',
'diana@example.com',
'eve@example.com',
'frank@example.com',
'grace@example.com',
'henry@example.com'
];
console.log(`[APP] Queuing ${users.length} email jobs...`);
for (const email of users) {
queue.add('send-email', { email });
}
// Wait until the queue has no queued or active work remaining.
while (queue.jobs.some((job) => job.status === 'queued' || job.status === 'processing')) {
await queue.sleep(250);
}
console.log('[APP] Queue is empty. Requesting graceful shutdown.');
queue.shutdown();
await workerPromise;
queue.printSummary();
console.log('[APP] Background job demo finished.');
}
main().catch((error) => {
console.error('[APP] Unexpected fatal error:', error);
process.exitCode = 1;
});
Conclusion
The most important lesson is that background processing is an architectural boundary, not simply a setTimeout around slow code. Once work is represented as jobs, the system can control concurrency, retry temporary failures, track execution state, and keep user-facing requests fast.
The in-memory queue used here is intentionally simple and should not be treated as durable production infrastructure. In a real Node.js application, a persistent queue such as BullMQ with Redis can provide delayed jobs, persistence, retry policies, job events, rate limiting, and distributed workers.
As your workload grows, focus on idempotency, observability, dead-letter handling, backpressure, and graceful deployment behavior. These details are what turn a background worker from a basic script into a reliable part of a production system.
Top comments (0)