Building a Redis-Powered Node.js Application: A Step-by-Step Guide
Redis is like that one friend who always seems to remember everything instantly. It's an in-memory data store that can significantly speed up your applications by reducing the need to repeatedly query your database. In this article, we'll build a Node.js Express application with Redis caching from scratch, explaining each step along the way.
Prerequisites
- Node.js and npm installed
- Basic understanding of Express.js
- A sense of humor (optional, but recommended)
Let's Get Right Into It
Installing Redis on Windows
Since Redis wasn't built with Windows in mind (they're not exactly on speaking terms), we need to use Windows Subsystem for Linux (WSL).
Step 1: Install WSL
Open PowerShell with administrator privileges and run:
wsl --install
This command installs WSL with Ubuntu. You might need to restart your computer, so save any cat videos you're watching for later.
Step 2: Set Up Ubuntu
After restart, a terminal will open asking you to create a username and password. Choose something memorable, unlike that "secure" password with 17 special characters you created last week and already forgot.
Step 3: Install Redis on Ubuntu
In your WSL terminal:
sudo apt-get update
sudo apt-get install redis-server
Step 4: Start Redis Server
sudo service redis-server start
To verify Redis is working:
redis-cli ping
If it responds with "PONG," congratulations! Redis is running. If not, well, we've all been there. Try restarting the service or check for error messages.
Building Our Application Step by Step
Step 1: Initialize Your Node.js Project
Create a new directory and initialize a Node.js project:
mkdir redis-test
cd redis-test
npm init -y
Step 2: Install Required Dependencies
npm install express mongoose dotenv redis
Step 3: Create Environment Variables
Create a .env file:
PORT=4000
MONGODB_URI=mongodb://localhost:27017/students-db
REDIS_URL=redis://localhost:6379
Step 4: Set Up Redis Connection
Create a directory for configuration files:
mkdir config
Now let's create the Redis connection file. This file will handle connecting to Redis and provide fallback functionality if Redis is unavailable.
Create config/redis.js:
const redis = require('redis');
// Create a Redis client with connection to Redis server
const createRedisClient = async () => {
try {
// Create Redis client
const client = redis.createClient({
url: process.env.REDIS_URL || 'redis://localhost:6379'
});
// Setup event handlers
client.on('error', (err) => {
console.error('Redis Error:', err);
});
client.on('connect', () => {
console.log('Redis connected');
});
client.on('reconnecting', () => {
console.log('Redis reconnecting...');
});
client.on('end', () => {
console.log('Redis connection closed');
});
// Connect to Redis
await client.connect();
return client;
} catch (err) {
console.error('Failed to create Redis client:', err);
// Return a mock client that stores data in memory as fallback
console.log('Using in-memory fallback for Redis');
const mockStorage = {};
return {
get: async (key) => mockStorage[key] || null,
set: async (key, value, options) => {
mockStorage[key] = value;
// Handle expiration if EX option provided
if (options && options.EX) {
setTimeout(() => {
delete mockStorage[key];
}, options.EX * 1000);
}
return 'OK';
},
del: async (key) => {
if (mockStorage[key]) {
delete mockStorage[key];
return 1;
}
return 0;
},
keys: async (pattern) => {
const regex = new RegExp('^' + pattern.replace('*', '.*') + '$');
return Object.keys(mockStorage).filter(key => regex.test(key));
},
// Add other Redis commands as needed for your application
hSet: async () => 'OK',
hGetAll: async () => ({}),
zAdd: async () => 1,
zRange: async () => [],
zRem: async () => 1,
exists: async () => 0
};
}
};
module.exports = { createRedisClient };
This setup gives us a robust Redis client that:
- Connects to our Redis server
- Handles errors and reconnection attempts
- Provides a fallback in-memory implementation if Redis is unavailable
Step 5: Create Main Application File
Now, let's create our main application file, building it step by step:
Create app.js:
// Load environment variables
require('dotenv').config();
// Import required packages
const express = require('express');
const mongoose = require('mongoose');
const { createRedisClient } = require('./config/redis');
// Initialize express app
const app = express();
const port = process.env.PORT || 4000;
// Global Redis client
let redisClient;
// Connect to databases and start server
async function startServer() {
try {
// Connect to Redis
redisClient = await createRedisClient();
// Connect to MongoDB
await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/students-db');
console.log('MongoDB connected');
// Start express server
app.listen(port, () => {
console.log(`Student CRUD API running on port ${port}`);
});
} catch (err) {
console.error('Failed to initialize connections:', err);
process.exit(1);
}
}
Step 6: Create Student Model
Add this code to your app.js:
// Student Schema
const studentSchema = new mongoose.Schema({
name: {
type: String,
required: true,
trim: true
},
email: {
type: String,
required: true,
unique: true,
trim: true,
lowercase: true
},
grade: {
type: String,
required: true,
trim: true
},
age: {
type: Number,
required: true,
min: 5
},
subjects: [{
type: String,
trim: true
}],
createdAt: {
type: Date,
default: Date.now
}
});
// Student Model
const Student = mongoose.model('Student', studentSchema);
// Middleware
app.use(express.json());
Step 7: Create Caching Middleware
This is where the magic happens. Let's add our caching middleware to app.js:
// Cache middleware
const cacheData = (expireTime = 3600) => {
return async (req, res, next) => {
// Skip caching for non-GET requests
if (req.method !== 'GET') {
return next();
}
// Create a cache key based on the full URL
const cacheKey = `students:${req.originalUrl}`;
try {
// Check if cache exists
const cachedData = await redisClient.get(cacheKey);
if (cachedData) {
console.log(`Cache hit for ${cacheKey}`);
return res.json(JSON.parse(cachedData));
}
console.log(`Cache miss for ${cacheKey}`);
// If not in cache, continue but modify res.json
res.originalJson = res.json;
res.json = function(data) {
// Store in cache before sending response
redisClient.set(cacheKey, JSON.stringify(data), { EX: expireTime })
.catch(err => console.error('Redis cache error:', err));
// Continue with the original response
return res.originalJson(data);
};
next();
} catch (err) {
console.error('Cache middleware error:', err);
next();
}
};
};
Let's break down what this middleware does:
- It only applies to GET requests because we don't typically want to cache modifications
- It creates a unique cache key based on the URL
- It checks if the data for that URL is already in Redis
- If found, it returns the cached data immediately (cache hit)
- If not found, it modifies the response to save the data to Redis before sending it (cache miss)
Step 8: Create Cache Invalidation Function
Next, we need a way to clear the cache when data changes:
// Clear cache helper
const clearCache = async (pattern) => {
try {
const keys = await redisClient.keys(pattern);
if (keys.length > 0) {
console.log(`Clearing cache keys matching: ${pattern}`);
await Promise.all(keys.map(key => redisClient.del(key)));
}
} catch (err) {
console.error('Error clearing cache:', err);
}
};
This function finds all Redis keys matching a pattern (like students:/api/students*) and deletes them, ensuring that when data changes, cached versions are cleared.
Step 9: Create API Routes with Caching
Now let's implement our API routes with Redis caching:
// Routes with caching
// 1. Get all students
app.get('/api/students', cacheData(60), async (req, res) => {
try {
const students = await Student.find({});
res.json(students);
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// 2. Get student by ID
app.get('/api/students/:id', cacheData(60), async (req, res) => {
try {
const student = await Student.findById(req.params.id);
if (!student) {
return res.status(404).json({ message: 'Student not found' });
}
res.json(student);
} catch (err) {
res.status(500).json({ error: err.message });
}
});
Notice how we're applying the cacheData(60) middleware to our GET routes? This means the data will be cached for 60 seconds. After that, a fresh copy will be retrieved from MongoDB.
Step 10: Create Routes That Invalidate Cache
When data changes, we need to invalidate our cache:
// 3. Create student
app.post('/api/students', async (req, res) => {
try {
const student = new Student(req.body);
await student.save();
// Clear the list cache when a new student is added
await clearCache('students:/api/students*');
res.status(201).json(student);
} catch (err) {
res.status(400).json({ error: err.message });
}
});
// 4. Update student
app.put('/api/students/:id', async (req, res) => {
try {
const student = await Student.findByIdAndUpdate(
req.params.id,
req.body,
{ new: true, runValidators: true }
);
if (!student) {
return res.status(404).json({ message: 'Student not found' });
}
// Clear both specific and list caches
await Promise.all([
clearCache(`students:/api/students/${req.params.id}`),
clearCache('students:/api/students*')
]);
res.json(student);
} catch (err) {
res.status(400).json({ error: err.message });
}
});
// 5. Delete student
app.delete('/api/students/:id', async (req, res) => {
try {
const student = await Student.findByIdAndDelete(req.params.id);
if (!student) {
return res.status(404).json({ message: 'Student not found' });
}
// Clear both specific and list caches
await Promise.all([
clearCache(`students:/api/students/${req.params.id}`),
clearCache('students:/api/students*')
]);
res.json({ message: 'Student deleted successfully' });
} catch (err) {
res.status(500).json({ error: err.message });
}
});
In each of these routes, we call clearCache() after modifying data to ensure users always see the most up-to-date information.
Step 11: Add More Advanced Routes
Let's add some more routes to show how we can cache filtered data:
// 6. Search students by name or email (with caching)
app.get('/api/students/search/:query', cacheData(30), async (req, res) => {
try {
const query = req.params.query;
const students = await Student.find({
$or: [
{ name: { $regex: query, $options: 'i' } },
{ email: { $regex: query, $options: 'i' } }
]
});
res.json(students);
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// 7. Get students by grade (with caching)
app.get('/api/students/grade/:grade', cacheData(60), async (req, res) => {
try {
const students = await Student.find({ grade: req.params.grade });
res.json(students);
} catch (err) {
res.status(500).json({ error: err.message });
}
});
We're using a shorter cache duration (30 seconds) for search results since they might change more frequently.
Step 12: Add Error Handler and Start the Server
Finally, let's add an error handler and start the server:
// Error handler
app.use((err, req, res, next) => {
console.error(err.stack);
res.status(500).json({ error: 'Something went wrong!' });
});
// Start the server
startServer();
module.exports = app;
Testing the Application
Now that we've built our application, let's test its caching capabilities:
- Start your server:
node app.js
- Create a new student:
curl -X POST http://localhost:4000/api/students -H "Content-Type: application/json" -d '{"name":"John Doe","email":"john@example.com","grade":"A","age":15,"subjects":["Math","Science"]}'
- Get all students (first request - cache miss):
curl http://localhost:4000/api/students
You should see "Cache miss" in your console.
- Get all students again (second request - cache hit):
curl http://localhost:4000/api/students
You should see "Cache hit" in your console and notice the response comes back much faster!
- Update a student, then get all students again to see cache invalidation in action:
curl -X PUT http://localhost:4000/api/students/[student-id] -H "Content-Type: application/json" -d '{"name":"John Smith"}'
curl http://localhost:4000/api/students
You should see "Cache miss" again since the cache was cleared.
The Benefits of Redis Caching
Blistering Speed: Redis operations happen in microseconds, while database queries can take milliseconds or more. That might not seem like much, but it adds up when you have thousands of users.
Reduced Database Load: Your database server can focus on important write operations instead of handling the same read requests over and over.
Scalable Architecture: By implementing Redis, you're already preparing your application for growth. As traffic increases, your caching layer will help maintain performance.
Improved User Experience: Faster response times lead to happier users. Nobody likes waiting for a website to load.
Common Redis Caching Pitfalls to Avoid
Caching Everything: Some data changes too frequently or is too personalized to benefit from caching.
Incorrect Cache Invalidation: Failing to clear the cache when data changes leads to stale data being served to users.
Using Too Long Expiration Times: Find the right balance between cache hits and data freshness.
Not Handling Redis Failures: Always have a fallback plan, like our in-memory cache replacement.
Conclusion
We've just built a complete Node.js application with Redis caching! By strategically implementing caching for our read operations and carefully invalidating the cache when data changes, we've created a system that can deliver fast responses while maintaining data accuracy.
Remember that caching isn't a silver bullet - it's a tool that, when used properly, can significantly improve your application's performance. Start with conservative cache durations and adjust based on your specific use case and data volatility.
As the wise computer science saying goes: "There are only two hard things in Computer Science: cache invalidation and naming things." Now you're well equipped to handle at least one of them!
Happy coding!
Complete Code
For reference, here's the complete app.js file we built throughout this tutorial:
require('dotenv').config();
const express = require('express');
const mongoose = require('mongoose');
const { createRedisClient } = require('./config/redis');
// Initialize express app
const app = express();
const port = process.env.PORT || 4000;
// Global Redis client
let redisClient;
// Connect to databases and start server
async function startServer() {
try {
// Connect to Redis
redisClient = await createRedisClient();
// Connect to MongoDB
await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/students-db');
console.log('MongoDB connected');
// Start express server
app.listen(port, () => {
console.log(`Student CRUD API running on port ${port}`);
});
} catch (err) {
console.error('Failed to initialize connections:', err);
process.exit(1);
}
}
// Student Schema
const studentSchema = new mongoose.Schema({
name: {
type: String,
required: true,
trim: true
},
email: {
type: String,
required: true,
unique: true,
trim: true,
lowercase: true
},
grade: {
type: String,
required: true,
trim: true
},
age: {
type: Number,
required: true,
min: 5
},
subjects: [{
type: String,
trim: true
}],
createdAt: {
type: Date,
default: Date.now
}
});
// Student Model
const Student = mongoose.model('Student', studentSchema);
// Middleware
app.use(express.json());
// Cache middleware
const cacheData = (expireTime = 3600) => {
return async (req, res, next) => {
// Skip caching for non-GET requests
if (req.method !== 'GET') {
return next();
}
// Create a cache key based on the full URL
const cacheKey = `students:${req.originalUrl}`;
try {
// Check if cache exists
const cachedData = await redisClient.get(cacheKey);
if (cachedData) {
console.log(`Cache hit for ${cacheKey}`);
return res.json(JSON.parse(cachedData));
}
console.log(`Cache miss for ${cacheKey}`);
// If not in cache, continue but modify res.json
res.originalJson = res.json;
res.json = function(data) {
// Store in cache before sending response
redisClient.set(cacheKey, JSON.stringify(data), { EX: expireTime })
.catch(err => console.error('Redis cache error:', err));
// Continue with the original response
return res.originalJson(data);
};
next();
} catch (err) {
console.error('Cache middleware error:', err);
next();
}
};
};
// Clear cache helper
const clearCache = async (pattern) => {
try {
const keys = await redisClient.keys(pattern);
if (keys.length > 0) {
console.log(`Clearing cache keys matching: ${pattern}`);
await Promise.all(keys.map(key => redisClient.del(key)));
}
} catch (err) {
console.error('Error clearing cache:', err);
}
};
// Routes with caching
// 1. Get all students
app.get('/api/students', cacheData(60), async (req, res) => {
try {
const students = await Student.find({});
res.json(students);
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// 2. Get student by ID
app.get('/api/students/:id', cacheData(60), async (req, res) => {
try {
const student = await Student.findById(req.params.id);
if (!student) {
return res.status(404).json({ message: 'Student not found' });
}
res.json(student);
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// 3. Create student
app.post('/api/students', async (req, res) => {
try {
const student = new Student(req.body);
await student.save();
// Clear the list cache when a new student is added
await clearCache('students:/api/students*');
res.status(201).json(student);
} catch (err) {
res.status(400).json({ error: err.message });
}
});
// 4. Update student
app.put('/api/students/:id', async (req, res) => {
try {
const student = await Student.findByIdAndUpdate(
req.params.id,
req.body,
{ new: true, runValidators: true }
);
if (!student) {
return res.status(404).json({ message: 'Student not found' });
}
// Clear both specific and list caches
await Promise.all([
clearCache(`students:/api/students/${req.params.id}`),
clearCache('students:/api/students*')
]);
res.json(student);
} catch (err) {
res.status(400).json({ error: err.message });
}
});
// 5. Delete student
app.delete('/api/students/:id', async (req, res) => {
try {
const student = await Student.findByIdAndDelete(req.params.id);
if (!student) {
return res.status(404).json({ message: 'Student not found' });
}
// Clear both specific and list caches
await Promise.all([
clearCache(`students:/api/students/${req.params.id}`),
clearCache('students:/api/students*')
]);
res.json({ message: 'Student deleted successfully' });
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// 6. Search students by name or email (with caching)
app.get('/api/students/search/:query', cacheData(30), async (req, res) => {
try {
const query = req.params.query;
const students = await Student.find({
$or: [
{ name: { $regex: query, $options: 'i' } },
{ email: { $regex: query, $options: 'i' } }
]
});
res.json(students);
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// 7. Get students by grade (with caching)
app.get('/api/students/grade/:grade', cacheData(60), async (req, res) => {
try {
const students = await Student.find({ grade: req.params.grade });
res.json(students);
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// Error handler
app.use((err, req, res, next) => {
console.error(err.stack);
res.status(500).json({ error: 'Something went wrong!' });
});
// Start the server
startServer();
module.exports = app;
And the complete redis.js file:
const redis = require('redis');
// Create a Redis client with connection to Redis server
const createRedisClient = async () => {
try {
// Create Redis client
const client = redis.createClient({
url: process.env.REDIS_URL || 'redis://localhost:6379'
});
// Setup event handlers
client.on('error', (err) => {
console.error('Redis Error:', err);
});
client.on('connect', () => {
console.log('Redis connected');
});
client.on('reconnecting', () => {
console.log('Redis reconnecting...');
});
client.on('end', () => {
console.log('Redis connection closed');
});
// Connect to Redis
await client.connect();
return client;
} catch (err) {
console.error('Failed to create Redis client:', err);
// Return a mock client that stores data in memory as fallback
console.log('Using in-memory fallback for Redis');
const mockStorage = {};
return {
get: async (key) => mockStorage[key] || null,
set: async (key, value, options) => {
mockStorage[key] = value;
// Handle expiration if EX option provided
if (options && options.EX) {
setTimeout(() => {
delete mockStorage[key];
}, options.EX * 1000);
}
return 'OK';
},
del: async (key) => {
if (mockStorage[key]) {
delete mockStorage[key];
return 1;
}
return 0;
},
keys: async (pattern) => {
const regex = new RegExp('^' + pattern.replace('*', '.*') + '$');
return Object.keys(mockStorage).filter(key => regex.test(key));
},
// Add other Redis commands as needed for your application
hSet: async () => 'OK',
hGetAll: async () => ({}),
zAdd: async () => 1,
zRange: async () => [],
zRem: async () => 1,
exists: async () => 0
};
}
};
module.exports = { createRedisClient };
Top comments (0)