The Problem With Ad-Hoc Degradation
Most graceful degradation starts as scattered if statements checking feature flags. Over time, nobody knows which routes have fallbacks, what those fallbacks do, or whether they've been tested. What you need is a single manifest that declares every degradable route, its fallback behavior, and its dependency requirements.
The Degradation Manifest
Define your degradation rules in YAML:
# degradation-manifest.yaml
defaults:
health_check_interval_ms: 5000
failure_threshold: 0.4
recovery_threshold: 0.15
hysteresis_seconds: 90
dependencies:
recommendations-api:
health_url: http://recommendations:8080/healthz
timeout_ms: 2000
analytics-collector:
health_url: http://analytics:8080/healthz
timeout_ms: 1000
personalization-service:
health_url: http://personalization:8080/healthz
timeout_ms: 2000
tiers:
best-effort:
priority: 1 # shed first
routes:
- path: /api/analytics/track
method: POST
fallback: drop # return 204, do nothing
requires: [analytics-collector]
- path: /api/ab/assign
method: GET
fallback: static
static_response: { "variant": "control", "source": "degraded" }
requires: [analytics-collector]
enhanced:
priority: 2
routes:
- path: /api/recommendations
method: GET
fallback: static
static_response: { "items": [], "source": "degraded" }
requires: [recommendations-api]
- path: /api/personalization
method: GET
fallback: static
static_response: { "profile": "default", "source": "degraded" }
requires: [personalization-service]
The Express Middleware
const express = require('express');
const fs = require('fs');
const yaml = require('js-yaml');
class DependencyMonitor {
constructor(name, config, defaults) {
this.name = name;
this.healthUrl = config.health_url;
this.timeoutMs = config.timeout_ms;
this.failureThreshold = defaults.failure_threshold;
this.recoveryThreshold = defaults.recovery_threshold;
this.window = []; // { timestamp, healthy }
this.windowMs = 30000;
}
record(healthy) {
const now = Date.now();
this.window.push({ timestamp: now, healthy });
const cutoff = now - this.windowMs;
this.window = this.window.filter(e => e.timestamp > cutoff);
}
get errorRate() {
if (this.window.length === 0) return 0;
const failures = this.window.filter(e => !e.healthy).length;
return failures / this.window.length;
}
get isFailed() {
return this.errorRate >= this.failureThreshold;
}
get isRecovered() {
return this.errorRate <= this.recoveryThreshold;
}
async check() {
try {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
const res = await fetch(this.healthUrl, { signal: controller.signal });
clearTimeout(timeout);
this.record(res.ok);
} catch {
this.record(false);
}
}
}
class DegradationEngine {
constructor(manifestPath) {
const raw = yaml.load(fs.readFileSync(manifestPath, 'utf8'));
this.defaults = raw.defaults;
this.simulationMode = false;
this.simulatedLevel = 0;
// Initialize dependency monitors
this.monitors = {};
for (const [name, config] of Object.entries(raw.dependencies)) {
this.monitors[name] = new DependencyMonitor(name, config, this.defaults);
}
// Build tier list sorted by priority
this.tiers = Object.entries(raw.tiers)
.map(([name, tier]) => ({
name,
priority: tier.priority,
routes: tier.routes,
}))
.sort((a, b) => a.priority - b.priority);
// Build route lookup
this.routeMap = new Map();
for (const tier of this.tiers) {
for (const route of tier.routes) {
const key = `${route.method.toUpperCase()}:${route.path}`;
this.routeMap.set(key, { ...route, tierName: tier.name, tierPriority: tier.priority });
}
}
// Current shed level (0 = normal, 1 = shed priority-1, etc.)
this.shedLevel = 0;
this.levelChangedAt = 0;
// Start health checking
this._interval = setInterval(() => this._evaluate(), this.defaults.health_check_interval_ms);
}
_evaluate() {
// Check all dependencies
for (const monitor of Object.values(this.monitors)) {
monitor.check();
}
if (this.simulationMode) return;
// Determine required shed level
let requiredLevel = 0;
for (const tier of this.tiers) {
const deps = new Set(tier.routes.flatMap(r => r.requires));
for (const depName of deps) {
if (this.monitors[depName]?.isFailed) {
requiredLevel = Math.max(requiredLevel, tier.priority);
}
}
}
// Escalate immediately
if (requiredLevel > this.shedLevel) {
this.shedLevel = requiredLevel;
this.levelChangedAt = Date.now();
return;
}
// Recover with hysteresis
if (requiredLevel < this.shedLevel) {
const elapsed = Date.now() - this.levelChangedAt;
if (elapsed >= this.defaults.hysteresis_seconds * 1000) {
const allRecovered = Object.values(this.monitors).every(m => m.isRecovered);
if (allRecovered) {
this.shedLevel = Math.max(0, this.shedLevel - 1);
this.levelChangedAt = Date.now();
}
}
}
}
get activeShedLevel() {
return this.simulationMode ? this.simulatedLevel : this.shedLevel;
}
middleware() {
return (req, res, next) => {
const key = `${req.method.toUpperCase()}:${req.path}`;
const route = this.routeMap.get(key);
if (route && route.tierPriority <= this.activeShedLevel) {
res.set('X-Degraded', 'true');
res.set('X-Degradation-Tier', route.tierName);
if (route.fallback === 'drop') {
return res.status(204).end();
}
if (route.fallback === 'static') {
return res.json(route.static_response);
}
}
res.set('X-Degradation-Level', String(this.activeShedLevel));
next();
};
}
statusHandler() {
return (req, res) => {
const shedTiers = this.tiers
.filter(t => t.priority <= this.activeShedLevel)
.map(t => t.name);
res.json({
shed_level: this.activeShedLevel,
simulation_mode: this.simulationMode,
shed_tiers: shedTiers,
dependencies: Object.fromEntries(
Object.entries(this.monitors).map(([name, m]) => [
name,
{ error_rate: Math.round(m.errorRate * 1000) / 1000, failed: m.isFailed },
])
),
});
};
}
simulationHandler() {
return (req, res) => {
const { enabled, level } = req.body;
if (typeof enabled === 'boolean') {
this.simulationMode = enabled;
}
if (typeof level === 'number') {
this.simulatedLevel = level;
}
res.json({
simulation_mode: this.simulationMode,
simulated_level: this.simulatedLevel,
actual_level: this.shedLevel,
});
};
}
}
// Usage
const app = express();
app.use(express.json());
const degradation = new DegradationEngine('./degradation-manifest.yaml');
app.use(degradation.middleware());
app.get('/degradation/status', degradation.statusHandler());
app.post('/degradation/simulate', degradation.simulationHandler());
// Your actual routes
app.get('/api/recommendations', (req, res) => {
// Only reached if not shed
res.json({ items: ['real', 'recommendations'] });
});
app.post('/api/analytics/track', (req, res) => {
// Only reached if not shed
res.status(200).json({ tracked: true });
});
app.listen(3000);
Simulation Mode
The simulation endpoint lets you test shed sequences without waiting for real failures:
# Enable simulation, shed tier 1 (best-effort)
curl -X POST http://localhost:3000/degradation/simulate \
-H 'Content-Type: application/json' \
-d '{"enabled": true, "level": 1}'
# Verify analytics tracking is shed
curl -v -X POST http://localhost:3000/api/analytics/track
# Returns 204 with X-Degraded: true
# Escalate to tier 2
curl -X POST http://localhost:3000/degradation/simulate \
-d '{"enabled": true, "level": 2}'
# Verify recommendations are shed
curl http://localhost:3000/api/recommendations
# Returns {"items":[], "source": "degraded"}
# Check status
curl http://localhost:3000/degradation/status
# Disable simulation
curl -X POST http://localhost:3000/degradation/simulate \
-d '{"enabled": false}'
Run this simulation as part of your CI pipeline. If someone adds a new route without a manifest entry, you'll catch it when the simulation test shows the route still serving live responses under degradation.
What This Buys You
The manifest is the source of truth for degradation behavior. New engineers can read it and understand the entire shed sequence. Product managers can review it without reading code. And the simulation mode means you can verify the degraded state before you're in an incident.
Top comments (0)