Design Chaos Engineering with Claude Code: Fault Injection, Resilience Testing, Auto-Stop

#claudecode #devops #kubernetes #testing

Introduction

Discovering failures after they happen is too late — chaos engineering intentionally injects faults to verify resilience. Let Claude Code design Chaos Mesh + Node.js fault injection patterns.

CLAUDE.md Rules

## Chaos Engineering Rules
- Staging first (production requires approval)
- Minimize blast radius (start with 1 service)
- Confirm monitoring before starting
- Kill switch: auto-stop on anomaly detection
- Success criteria: error rate < 0.1%, P99 < 500ms

Generated Implementation

# Chaos Mesh: network delay injection
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: order-service-delay
spec:
  action: delay
  mode: one
  selector:
    labelSelectors:
      app: order-service
  delay:
    latency: "200ms"
    jitter: "50ms"
  duration: "10m"

---
# Pod kill (Chaos Monkey)
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
spec:
  action: pod-kill
  mode: one
  selector:
    labelSelectors:
      app: myapp
  scheduler:
    cron: "@every 5m"

// Auto-stop on high error rate
export class ChaosRunner {
  async runExperiment(experiment: ChaosExperiment) {
    const monitorInterval = setInterval(async () => {
      const sample = await this.collectMetrics();

      // Auto-stop: error rate > 1%
      if (sample.errorRate > 0.01) {
        logger.error('High error rate! Stopping experiment.');
        this.stopRequested = true;
      }
    }, 10_000);

    try {
      await this.waitForDurationOrStop(experiment.duration);
    } finally {
      clearInterval(monitorInterval);
      await this.stopExperiment(experimentHandle); // Always stop
    }
  }
}

// Node.js fault injection for CI testing
export function injectFaults(config: { errorRate: number; latencyMs?: number }) {
  return async (url: string, options: RequestInit) => {
    if (config.latencyMs) await sleep(config.latencyMs);
    if (Math.random() < config.errorRate) throw new Error('Injected fault');
    return fetch(url, options);
  };
}