🧠 Building a Multi-Agent System with Long-Term Memory: A Technical Guide
A practical guide to building AI agents with memory for real-world applications
Introduction
Multi-agent systems are becoming increasingly important in AI applications. This guide walks you through building a production-ready multi-agent system with long-term memory, inspired by Google's approach to personalized AI.
We'll build a system where specialized agents work together, each with their own memory and capabilities.
🏗️ System Architecture
Overview
┌─────────────────────────────────────────────────────────────────────────────┐
│ MULTI-AGENT SYSTEM │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ ┌─────────────────────────────────────────────────────────────────────┐ │
│ │ ORCHESTRATOR AGENT │ │
│ │ • Coordinates all other agents │ │
│ │ • Maintains conversation history │ │
│ │ • Manages agent lifecycle │ │
│ └─────────────────────────────┬───────────────────────────────────────┘ │
│ │ │
│ ┌───────────────────────┼───────────────────────┐ │
│ ▼ ▼ ▼ │
│ ┌─────────────┐ ┌─────────────────────┐ ┌─────────────────────┐ │
│ │ INVESTOR │ │ LEGAL │ │ TECHNICAL │ │
│ │ AGENT │ │ AGENT │ │ AGENT │ │
│ └─────────────┘ └─────────────────────┘ └─────────────────────┘ │
│ │
│ ┌─────────────────────────────────────────────────────────────────────┐ │
│ │ MEMORY LAYER │ │
│ │ • Short-term: Conversation context │ │
│ │ • Long-term: Vector database for semantic search │ │
│ │ • Episodic: Event history │ │
│ └─────────────────────────────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
text
💻 Implementation
1. Base Agent Class
The foundation for all agents:
javascript
// src/agents/base-agent.js
class BaseAgent {
constructor(config) {
this.name = config.name;
this.role = config.role;
this.memory = config.memory || null;
this.lastTask = null;
this.stats = {
tasksProcessed: 0,
errors: 0,
lastActivity: null
};
}
async process(data, context = {}) {
try {
this.stats.tasksProcessed++;
this.stats.lastActivity = new Date();
this.lastTask = { data, context, timestamp: new Date() };
const result = await this.execute(data, context);
return {
success: true,
agent: this.name,
result,
timestamp: new Date().toISOString()
};
} catch (error) {
this.stats.errors++;
return {
success: false,
agent: this.name,
error: error.message,
timestamp: new Date().toISOString()
};
}
}
async execute(data, context) {
throw new Error('execute() must be implemented by subclass');
}
getStatus() {
return {
name: this.name,
role: this.role,
stats: this.stats,
lastTask: this.lastTask
};
}
}
module.exports = { BaseAgent };
2. Memory System
Three-tier memory architecture:
javascript
// src/agents/memory/index.js
class AgentMemory {
constructor() {
this.shortTerm = [];
this.longTerm = new Map();
this.episodic = [];
this.maxShortTerm = 50;
}
// === SHORT-TERM MEMORY ===
// Stores recent conversation context
addShortTerm(entry) {
this.shortTerm.push({
...entry,
timestamp: new Date().toISOString()
});
if (this.shortTerm.length > this.maxShortTerm) {
this.shortTerm.shift();
}
}
getShortTerm(limit = 10) {
return this.shortTerm.slice(-limit);
}
// === LONG-TERM MEMORY ===
// Persistent storage with semantic search
async storeLongTerm(key, value, metadata = {}) {
this.longTerm.set(key, {
value,
metadata,
timestamp: new Date().toISOString()
});
return true;
}
async retrieveLongTerm(key) {
return this.longTerm.get(key) || null;
}
async searchLongTerm(query) {
const results = [];
for (const [key, data] of this.longTerm) {
if (key.includes(query) || JSON.stringify(data.value).includes(query)) {
results.push({ key, ...data });
}
}
return results.slice(0, 10);
}
// === EPISODIC MEMORY ===
// Event history for pattern recognition
addEvent(event) {
this.episodic.push({
...event,
timestamp: new Date().toISOString()
});
}
getEvents(type, limit = 10) {
const filtered = this.episodic.filter(e => e.type === type);
return filtered.slice(-limit);
}
getStats() {
return {
shortTermSize: this.shortTerm.length,
longTermSize: this.longTerm.size,
episodicSize: this.episodic.length
};
}
}
module.exports = { AgentMemory };
3. Investor Agent Example
javascript
// src/agents/investor-agent.js
const { BaseAgent } = require('./base-agent');
const { AgentMemory } = require('./memory');
class InvestorAgent extends BaseAgent {
constructor() {
super({
name: 'Investor Agent',
role: 'Investor Onboarding & KYC'
});
this.memory = new AgentMemory();
this.kycStatus = new Map();
}
async execute(data, context) {
const { action, email, name, netWorth, documents } = data;
switch (action) {
case 'onboard':
return this.onboardInvestor(email, name, netWorth, documents, context);
case 'verify':
return this.verifyInvestor(email, context);
case 'portfolio':
return this.getPortfolio(email, context);
case 'history':
return this.getHistory(email, context);
default:
throw new Error(`Unknown action: ${action}`);
}
}
async onboardInvestor(email, name, netWorth, documents, context) {
// Check if investor exists
const existing = await this.memory.retrieveLongTerm(`investor:${email}`);
if (existing) {
this.memory.addShortTerm({
type: 'investor-onboard',
action: 'returning',
email,
context
});
return {
status: 'existing',
message: 'Investor already onboarded',
investor: existing.value,
history: this.memory.getEvents('investor')
};
}
// New investor - perform KYC
const kycResult = await this.performKYC(name, netWorth, documents);
// Store in memory
const investorData = {
email,
name,
netWorth,
kycStatus: kycResult.status,
onboardedAt: new Date().toISOString(),
documents: documents || []
};
await this.memory.storeLongTerm(`investor:${email}`, investorData, {
type: 'investor',
status: kycResult.status
});
this.memory.addEvent({
type: 'investor',
action: 'onboard',
email,
status: kycResult.status
});
this.memory.addShortTerm({
type: 'investor-onboard',
action: 'new',
email,
status: kycResult.status,
context
});
return {
status: kycResult.status,
message: kycResult.message,
investorId: `inv_${Date.now()}`,
nextSteps: kycResult.nextSteps || ['Complete verification']
};
}
async performKYC(name, netWorth, documents) {
// Simulate KYC verification
const status = netWorth >= 1000000 ? 'verified' : 'pending';
return {
status,
message: status === 'verified' ? 'KYC approved' : 'KYC needs review',
nextSteps: status === 'verified' ? ['Ready to invest'] : ['Submit additional documents']
};
}
async verifyInvestor(email, context) {
const investor = await this.memory.retrieveLongTerm(`investor:${email}`);
if (!investor) {
return {
status: 'not-found',
message: 'Investor not found'
};
}
this.memory.addShortTerm({
type: 'investor-verify',
email,
context
});
return {
status: investor.value.kycStatus,
investor: investor.value,
history: this.memory.getEvents('investor', 5)
};
}
async getPortfolio(email, context) {
// Portfolio retrieval logic
return {
status: 'success',
portfolio: {
tokens: [],
totalValue: 0,
lastUpdated: new Date().toISOString()
}
};
}
async getHistory(email, context) {
const history = this.memory.getEvents('investor', 20);
return {
status: 'success',
history: history.filter(e => e.email === email || !e.email)
};
}
}
module.exports = { InvestorAgent };
4. Legal Agent Example
javascript
// src/agents/legal-agent.js
const { BaseAgent } = require('./base-agent');
const { AgentMemory } = require('./memory');
class LegalAgent extends BaseAgent {
constructor() {
super({
name: 'Legal Agent',
role: 'Compliance & Documentation'
});
this.memory = new AgentMemory();
this.complianceRules = this.loadComplianceRules();
}
loadComplianceRules() {
return {
singapore: {
accreditedInvestor: { minNetWorth: 2000000, minIncome: 300000 },
smallOffer: { maxAmount: 5000000, maxInvestors: 50 },
prospectusExemption: true
},
hongKong: {
professionalInvestor: { minPortfolio: 8000000 },
minInvestment: 500000,
smallOffer: { maxAmount: 5000000 }
}
};
}
async execute(data, context) {
const { action, tokenData, jurisdiction, documents } = data;
switch (action) {
case 'review-token':
return this.reviewToken(tokenData, jurisdiction, context);
case 'check-compliance':
return this.checkCompliance(tokenData, jurisdiction, context);
case 'analyze-document':
return this.analyzeDocument(documents, context);
case 'get-requirements':
return this.getRequirements(jurisdiction, context);
default:
throw new Error(`Unknown action: ${action}`);
}
}
async reviewToken(tokenData, jurisdiction, context) {
const compliance = await this.checkCompliance(tokenData, jurisdiction, context);
const docs = await this.analyzeDocument(tokenData.documents, context);
const result = {
compliant: compliance.status === 'compliant',
issues: compliance.issues,
recommendations: compliance.recommendations,
documentStatus: docs.status,
jurisdiction: jurisdiction || 'singapore'
};
await this.memory.storeLongTerm(
`token-review:${tokenData.symbol || Date.now()}`,
result,
{ type: 'legal-review', jurisdiction }
);
this.memory.addEvent({
type: 'legal',
action: 'review-token',
symbol: tokenData.symbol,
status: result.compliant ? 'passed' : 'needs-review'
});
return result;
}
async checkCompliance(tokenData, jurisdiction, context) {
const rules = this.complianceRules[jurisdiction || 'singapore'];
const issues = [];
const recommendations = [];
// Check minimum investment
if (tokenData.minInvestment < rules.minInvestment) {
issues.push('Minimum investment below threshold');
recommendations.push(`Increase minimum investment to S$${rules.minInvestment}`);
}
// Check small offer exemption
if (tokenData.totalRaised > rules.smallOffer.maxAmount) {
issues.push('Exceeds small offer exemption limit');
recommendations.push('Consider splitting into multiple offerings');
}
// Check document completeness
if (!tokenData.documents || tokenData.documents.length < 3) {
issues.push('Insufficient legal documentation');
recommendations.push('Provide: Title deed, Valuation report, SPV incorporation');
}
return {
status: issues.length === 0 ? 'compliant' : 'needs-review',
issues,
recommendations
};
}
async analyzeDocument(documents, context) {
if (!documents || documents.length === 0) {
return {
status: 'missing',
message: 'No documents provided for analysis'
};
}
const analyzed = documents.map(doc => ({
name: doc.name || 'Unnamed document',
type: doc.type || 'unknown',
status: 'processed',
contentLength: doc.content?.length || 0
}));
this.memory.addShortTerm({
type: 'document-analysis',
count: documents.length,
context
});
return {
status: 'processed',
documents: analyzed,
summary: `${documents.length} documents analyzed`
};
}
async getRequirements(jurisdiction, context) {
const rules = this.complianceRules[jurisdiction || 'singapore'];
return {
jurisdiction: jurisdiction || 'singapore',
requirements: rules,
summary: this.summarizeRequirements(rules)
};
}
summarizeRequirements(rules) {
const parts = [];
if (rules.accreditedInvestor) {
parts.push(`Accredited investor: Net worth > S$${rules.accreditedInvestor.minNetWorth}`);
}
if (rules.minInvestment) {
parts.push(`Minimum investment: S$${rules.minInvestment}`);
}
if (rules.smallOffer) {
parts.push(`Small offer limit: S$${rules.smallOffer.maxAmount}`);
}
return parts.join('; ');
}
}
module.exports = { LegalAgent };
5. Technical Agent Example
javascript
// src/agents/technical-agent.js
const { BaseAgent } = require('./base-agent');
const { AgentMemory } = require('./memory');
class TechnicalAgent extends BaseAgent {
constructor() {
super({
name: 'Technical Agent',
role: 'Token Deployment & Blockchain Operations'
});
this.memory = new AgentMemory();
this.deployments = new Map();
}
async execute(data, context) {
const { action, tokenConfig, contractAddress } = data;
switch (action) {
case 'deploy-token':
return this.deployToken(tokenConfig, context);
case 'audit-contract':
return this.auditContract(contractAddress, context);
case 'get-token-info':
return this.getTokenInfo(contractAddress, context);
case 'verify-contract':
return this.verifyContract(contractAddress, context);
default:
throw new Error(`Unknown action: ${action}`);
}
}
async deployToken(tokenConfig, context) {
// Check if token already exists
const existing = await this.memory.retrieveLongTerm(`token:${tokenConfig.symbol}`);
if (existing) {
return {
status: 'exists',
message: `Token ${tokenConfig.symbol} already deployed`,
address: existing.value.address
};
}
// Simulate deployment
const deployment = {
address: `0x${Array.from({length: 40}, () => Math.floor(Math.random() * 16).toString(16)).join('')}`,
txHash: `0x${Array.from({length: 64}, () => Math.floor(Math.random() * 16).toString(16)).join('')}`,
gasUsed: Math.floor(Math.random() * 1000000) + 500000,
blockNumber: Math.floor(Math.random() * 10000000) + 10000000
};
// Store deployment
await this.memory.storeLongTerm(`token:${tokenConfig.symbol}`, {
...tokenConfig,
...deployment,
deployedAt: new Date().toISOString()
}, {
type: 'deployment',
symbol: tokenConfig.symbol,
status: 'success'
});
this.memory.addEvent({
type: 'technical',
action: 'deploy',
symbol: tokenConfig.symbol,
address: deployment.address
});
this.deployments.set(tokenConfig.symbol, deployment);
return {
status: 'success',
...deployment,
message: `Token ${tokenConfig.symbol} deployed successfully`
};
}
async auditContract(contractAddress, context) {
// Check if already audited
const audit = await this.memory.retrieveLongTerm(`audit:${contractAddress}`);
if (audit) {
return {
status: 'cached',
...audit.value,
message: 'Audit results retrieved from cache'
};
}
// Simulate audit
const result = {
score: Math.floor(Math.random() * 30) + 70, // 70-100
status: Math.random() > 0.2 ? 'passed' : 'needs-review',
vulnerabilities: Math.random() > 0.5 ? [] : [
{ severity: 'low', description: 'Potential gas optimization' }
],
recommendations: [
'Consider adding more tests',
'Review access control'
],
auditedAt: new Date().toISOString()
};
await this.memory.storeLongTerm(`audit:${contractAddress}`, result, {
type: 'audit',
status: result.status
});
return {
status: 'success',
...result
};
}
async getTokenInfo(contractAddress, context) {
return {
status: 'success',
token: {
address: contractAddress,
symbol: 'TST',
name: 'Test Token',
decimals: 18,
totalSupply: 1000000,
price: 1.00
}
};
}
async verifyContract(contractAddress, context) {
return {
status: 'success',
verified: true,
message: 'Contract verified on Etherscan',
link: `https://sepolia.etherscan.io/address/${contractAddress}`
};
}
}
module.exports = { TechnicalAgent };
6. Orchestrator
javascript
// src/agents/orchestrator.js
const EventEmitter = require('events');
const { AgentMemory } = require('./memory');
class Orchestrator extends EventEmitter {
constructor() {
super();
this.agents = new Map();
this.memory = new AgentMemory();
this.activeTasks = new Map();
this.taskHistory = [];
}
registerAgent(name, agent) {
this.agents.set(name, agent);
this.memory.addEvent({
type: 'orchestrator',
action: 'register-agent',
agent: name
});
console.log(`✅ Agent registered: ${name}`);
}
async executeTask(task) {
const { type, data, context = {} } = task;
const taskId = `${type}:${Date.now()}`;
const agent = this.selectAgent(type);
if (!agent) {
throw new Error(`No agent available for task: ${type}`);
}
const history = this.memory.getEvents(type, 5);
this.activeTasks.set(taskId, { task, status: 'running', startedAt: new Date() });
try {
const result = await agent.process(data, { ...context, history });
this.activeTasks.set(taskId, { task, status: 'completed', result, completedAt: new Date() });
this.taskHistory.push({ taskId, type, result, timestamp: new Date() });
this.memory.addEvent({
type: 'orchestrator',
action: 'task-complete',
taskId,
status: 'success'
});
this.emit('task-complete', { taskId, result });
return result;
} catch (error) {
this.activeTasks.set(taskId, { task, status: 'failed', error: error.message, completedAt: new Date() });
this.memory.addEvent({
type: 'orchestrator',
action: 'task-failed',
taskId,
error: error.message
});
this.emit('task-failed', { taskId, error: error.message });
throw error;
}
}
selectAgent(taskType) {
const mapping = {
'investor': 'Investor Agent',
'legal': 'Legal Agent',
'compliance': 'Legal Agent',
'technical': 'Technical Agent',
'deployment': 'Technical Agent',
'kyc': 'Investor Agent',
'audit': 'Technical Agent'
};
const agentName = mapping[taskType];
return this.agents.get(agentName) || null;
}
async getStatus() {
const agentStatus = {};
for (const [name, agent] of this.agents) {
agentStatus[name] = agent.getStatus();
}
return {
agents: agentStatus,
activeTasks: this.activeTasks.size,
totalTasks: this.taskHistory.length,
memory: this.memory.getStats()
};
}
getTaskStatus(taskId) {
return this.activeTasks.get(taskId) || null;
}
getHistory(limit = 20) {
return this.taskHistory.slice(-limit);
}
}
module.exports = { Orchestrator };
🚀 Integration with Express
javascript
// server.js
const express = require('express');
const { Orchestrator } = require('./agents/orchestrator');
const { InvestorAgent } = require('./agents/investor-agent');
const { LegalAgent } = require('./agents/legal-agent');
const { TechnicalAgent } = require('./agents/technical-agent');
const app = express();
const orchestrator = new Orchestrator();
// Register agents
orchestrator.registerAgent('Investor Agent', new InvestorAgent());
orchestrator.registerAgent('Legal Agent', new LegalAgent());
orchestrator.registerAgent('Technical Agent', new TechnicalAgent());
// API endpoints
app.post('/api/agents/task', async (req, res) => {
try {
const result = await orchestrator.executeTask(req.body);
res.json({
success: true,
data: result
});
} catch (error) {
res.status(500).json({
success: false,
error: error.message
});
}
});
app.get('/api/agents/status', async (req, res) => {
try {
const status = await orchestrator.getStatus();
res.json({
success: true,
data: status
});
} catch (error) {
res.status(500).json({
success: false,
error: error.message
});
}
});
app.get('/api/agents/history', async (req, res) => {
try {
const history = orchestrator.getHistory(50);
res.json({
success: true,
data: history
});
} catch (error) {
res.status(500).json({
success: false,
error: error.message
});
}
});
app.listen(3000, () => {
console.log('🚀 Server running on port 3000');
});
🧪 Testing
javascript
// tests/agents/test-agents.js
const { Orchestrator } = require('../../src/agents/orchestrator');
const { InvestorAgent } = require('../../src/agents/investor-agent');
const { LegalAgent } = require('../../src/agents/legal-agent');
const { TechnicalAgent } = require('../../src/agents/technical-agent');
async function testAgents() {
console.log('🧪 Testing Multi-Agent System...\n');
const orchestrator = new Orchestrator();
orchestrator.registerAgent('Investor Agent', new InvestorAgent());
orchestrator.registerAgent('Legal Agent', new LegalAgent());
orchestrator.registerAgent('Technical Agent', new TechnicalAgent());
// Test Investor Agent
console.log('📋 Testing Investor Agent...');
const investorResult = await orchestrator.executeTask({
type: 'investor',
data: {
action: 'onboard',
email: 'test@example.com',
name: 'Test User',
netWorth: 2000000,
documents: []
},
context: { source: 'test' }
});
console.log('Result:', investorResult.success ? '✅' : '❌', investorResult);
// Test Legal Agent
console.log('\n📋 Testing Legal Agent...');
const legalResult = await orchestrator.executeTask({
type: 'legal',
data: {
action: 'review-token',
tokenData: {
symbol: 'TST',
name: 'Test Token',
minInvestment: 10000,
totalRaised: 100000,
documents: [{ name: 'doc1.pdf', type: 'legal', content: 'test content' }]
},
jurisdiction: 'singapore'
},
context: { source: 'test' }
});
console.log('Result:', legalResult.success ? '✅' : '❌', legalResult);
// Test Technical Agent
console.log('\n📋 Testing Technical Agent...');
const techResult = await orchestrator.executeTask({
type: 'technical',
data: {
action: 'deploy-token',
tokenConfig: {
symbol: 'TST2',
name: 'Test Token 2',
supply: 10000,
price: 100
}
},
context: { source: 'test' }
});
console.log('Result:', techResult.success ? '✅' : '❌', techResult);
// Get status
console.log('\n📊 System Status:');
const status = await orchestrator.getStatus();
console.log(JSON.stringify(status, null, 2));
console.log('\n✅ Tests completed!');
}
testAgents().catch(console.error);
📊 Best Practices
Practice Description
Memory Tiers Use short-term for context, long-term for persistence, episodic for events
Agent Specialization Each agent should have a single, well-defined responsibility
Orchestration Centralized orchestrator for task routing and coordination
Error Handling Graceful failure with retry logic and fallbacks
Monitoring Track agent status, tasks, and performance metrics
Testing Unit tests for each agent and integration tests for the system
🔧 Dependencies
json
{
"dependencies": {
"express": "^4.18.0",
"pinecone-client": "^1.0.0",
"chromadb": "^1.0.0",
"openai": "^4.0.0",
"langchain": "^0.1.0"
}
}
📚 Resources
Google's Multi-Agent Research
LangChain Agents
Pinecone Vector Database
OpenAI Function Calling
🔗 Connect With Me
Platform Link
GitHub github.com/DanielIoni-creator
Twitter/X @MyZubster
Dev.to dev.to/danielioni
Built with ❤️ by Daniel Ioni
Top comments (0)