How to Monitor Success Rates of Language Model Calls in n8n

1-- PostgreSQL: Create the LLM call log table
2CREATE TABLE IF NOT EXISTS llm_call_logs (
3  id SERIAL PRIMARY KEY,
4  workflow_id VARCHAR(255) NOT NULL,
5  workflow_name VARCHAR(255),
6  node_name VARCHAR(255),
7  provider VARCHAR(50) NOT NULL,       -- openai, anthropic, mistral, cohere
8  model VARCHAR(100),
9  status VARCHAR(20) NOT NULL,          -- success, error, timeout, rate_limited
10  error_message TEXT,
11  error_code VARCHAR(10),
12  latency_ms INTEGER,
13  prompt_tokens INTEGER,
14  completion_tokens INTEGER,
15  total_tokens INTEGER,
16  session_id VARCHAR(255),
17  created_at TIMESTAMP DEFAULT NOW()
18);
19
20CREATE INDEX idx_llm_logs_created ON llm_call_logs(created_at);
21CREATE INDEX idx_llm_logs_status ON llm_call_logs(status);
22CREATE INDEX idx_llm_logs_provider ON llm_call_logs(provider);

1// Code node — JavaScript
2// Extract metrics from a successful LLM call
3// Place after your OpenAI/Claude/Mistral node
4
5const item = $input.first();
6const startTime = $('Webhook').first().json._startTime || Date.now();
7const latencyMs = Date.now() - startTime;
8
9// Extract token usage (format varies by provider)
10const usage = item.json.usage || item.json.message?.usage || {};
11
12return [{
13  json: {
14    workflow_id: $workflow.id,
15    workflow_name: $workflow.name,
16    node_name: $node.name,
17    provider: 'openai', // Change per provider
18    model: item.json.model || 'unknown',
19    status: 'success',
20    error_message: null,
21    error_code: null,
22    latency_ms: latencyMs,
23    prompt_tokens: usage.prompt_tokens || usage.input_tokens || 0,
24    completion_tokens: usage.completion_tokens || usage.output_tokens || 0,
25    total_tokens: (usage.prompt_tokens || usage.input_tokens || 0) + (usage.completion_tokens || usage.output_tokens || 0),
26    session_id: $('Webhook').first().json.sessionId || null
27  }
28}];

1// Code node in Error Workflow — JavaScript
2// Categorize and log LLM failures
3
4const errorData = $input.first().json;
5const errorMessage = errorData.execution?.error?.message || 'Unknown error';
6const nodeName = errorData.execution?.lastNodeExecuted || 'Unknown';
7
8// Categorize the error
9let status = 'error';
10let errorCode = '';
11
12if (errorMessage.includes('ETIMEDOUT') || errorMessage.includes('timeout')) {
13  status = 'timeout';
14  errorCode = '408';
15} else if (errorMessage.includes('429') || errorMessage.includes('rate limit')) {
16  status = 'rate_limited';
17  errorCode = '429';
18} else if (errorMessage.includes('401') || errorMessage.includes('Unauthorized')) {
19  status = 'auth_error';
20  errorCode = '401';
21} else if (errorMessage.includes('500') || errorMessage.includes('Internal Server')) {
22  status = 'server_error';
23  errorCode = '500';
24} else if (errorMessage.includes('529') || errorMessage.includes('overloaded')) {
25  status = 'server_error';
26  errorCode = '529';
27}
28
29return [{
30  json: {
31    workflow_id: errorData.workflow?.id || 'unknown',
32    workflow_name: errorData.workflow?.name || 'unknown',
33    node_name: nodeName,
34    provider: nodeName.toLowerCase().includes('openai') ? 'openai'
35      : nodeName.toLowerCase().includes('claude') ? 'anthropic'
36      : nodeName.toLowerCase().includes('mistral') ? 'mistral'
37      : 'unknown',
38    model: 'unknown',
39    status,
40    error_message: errorMessage.substring(0, 500),
41    error_code: errorCode,
42    latency_ms: null,
43    prompt_tokens: 0,
44    completion_tokens: 0,
45    total_tokens: 0,
46    session_id: null
47  }
48}];

1-- SQL query for the Postgres node
2-- Daily success rate aggregation
3SELECT
4  provider,
5  COUNT(*) as total_calls,
6  COUNT(*) FILTER (WHERE status = 'success') as successful_calls,
7  ROUND(
8    COUNT(*) FILTER (WHERE status = 'success')::numeric / COUNT(*)::numeric * 100, 2
9  ) as success_rate,
10  COUNT(*) FILTER (WHERE status = 'timeout') as timeouts,
11  COUNT(*) FILTER (WHERE status = 'rate_limited') as rate_limits,
12  COUNT(*) FILTER (WHERE status = 'error') as errors,
13  ROUND(AVG(latency_ms) FILTER (WHERE status = 'success'), 0) as avg_latency_ms,
14  SUM(total_tokens) as total_tokens_used
15FROM llm_call_logs
16WHERE created_at >= NOW() - INTERVAL '24 hours'
17GROUP BY provider
18ORDER BY success_rate ASC;

1// Code node — JavaScript
2// Check success rates and generate alert if needed
3
4const items = $input.all();
5const THRESHOLD = 95; // Alert if success rate drops below 95%
6
7const alerts = [];
8const summary = [];
9
10for (const item of items) {
11  const rate = parseFloat(item.json.success_rate);
12  const provider = item.json.provider;
13
14  summary.push(
15    `${provider}: ${rate}% success (${item.json.total_calls} calls, ${item.json.avg_latency_ms}ms avg latency)`
16  );
17
18  if (rate < THRESHOLD) {
19    alerts.push(
20      `⚠ ${provider}: ${rate}% success rate (${item.json.errors} errors, ${item.json.timeouts} timeouts, ${item.json.rate_limits} rate limits)`
21    );
22  }
23}
24
25return [{
26  json: {
27    hasAlerts: alerts.length > 0,
28    alertMessage: alerts.length > 0
29      ? `LLM Success Rate Alert\n\n${alerts.join('\n')}\n\nFull Summary:\n${summary.join('\n')}`
30      : null,
31    summaryMessage: `Daily LLM Report\n\n${summary.join('\n')}`,
32    alertCount: alerts.length
33  }
34}];