How to Build a Content Moderation Tool with V0

1-- Content items to moderate
2create table content_items (
3  id uuid primary key default gen_random_uuid(),
4  author_id uuid references auth.users not null,
5  content_type text check (content_type in ('post','comment','image','profile')) not null,
6  content_text text,
7  content_url text,
8  status text check (status in ('pending','approved','rejected','escalated')) default 'pending',
9  auto_score numeric,
10  auto_categories text[] default '{}',
11  source_table text,
12  source_id uuid,
13  created_at timestamptz default now()
14);
15
16-- Moderation actions (audit trail)
17create table moderation_actions (
18  id uuid primary key default gen_random_uuid(),
19  content_id uuid references content_items not null,
20  moderator_id uuid references auth.users not null,
21  action text check (action in ('approve','reject','escalate','warn','ban')) not null,
22  reason text,
23  created_at timestamptz default now()
24);
25
26-- Custom moderation rules
27create table moderation_rules (
28  id uuid primary key default gen_random_uuid(),
29  name text not null,
30  pattern text not null,
31  action text check (action in ('flag','auto_reject')) not null,
32  is_active boolean default true,
33  created_at timestamptz default now()
34);
35
36-- User strike tracking
37create table user_strikes (
38  user_id uuid primary key references auth.users,
39  strike_count int default 0,
40  is_banned boolean default false,
41  last_strike_at timestamptz
42);
43
44-- Trigger: increment strikes on warn/ban action
45create or replace function handle_moderation_action()
46returns trigger as $$
47begin
48  if NEW.action in ('warn', 'ban') then
49    -- Get the content author
50    declare
51      v_author_id uuid;
52    begin
53      select author_id into v_author_id
54      from content_items where id = NEW.content_id;
55
56      -- Upsert user strikes
57      insert into user_strikes (user_id, strike_count, last_strike_at)
58      values (v_author_id, 1, now())
59      on conflict (user_id) do update set
60        strike_count = user_strikes.strike_count + 1,
61        last_strike_at = now();
62
63      -- Auto-ban at 3 strikes
64      update user_strikes
65      set is_banned = true
66      where user_id = v_author_id and strike_count >= 3;
67    end;
68  end if;
69  return NEW;
70end;
71$$ language plpgsql security definer;
72
73create trigger on_moderation_action
74  after insert on moderation_actions
75  for each row execute function handle_moderation_action();
76
77-- RLS
78alter table content_items enable row level security;
79alter table moderation_actions enable row level security;
80alter table moderation_rules enable row level security;
81alter table user_strikes enable row level security;
82
83-- Only moderators can access (use a role check in practice)
84create policy "Moderators read content" on content_items for select using (true);
85create policy "Moderators update content" on content_items for update using (true);
86create policy "Moderators read actions" on moderation_actions for select using (true);
87create policy "Moderators create actions" on moderation_actions for insert with check (true);
88create policy "Moderators manage rules" on moderation_rules for all using (true);
89create policy "Moderators read strikes" on user_strikes for select using (true);

1// app/api/moderate/auto/route.ts
2import { NextRequest, NextResponse } from 'next/server';
3import { createClient } from '@supabase/supabase-js';
4
5const supabase = createClient(
6  process.env.NEXT_PUBLIC_SUPABASE_URL!,
7  process.env.SUPABASE_SERVICE_ROLE_KEY!
8);
9
10const AUTO_APPROVE_THRESHOLD = 0.3;
11const WEBHOOK_SECRET = process.env.MODERATION_WEBHOOK_SECRET!;
12
13export async function POST(request: NextRequest) {
14  // Validate webhook secret
15  const authHeader = request.headers.get('authorization');
16  if (authHeader !== `Bearer ${WEBHOOK_SECRET}`) {
17    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
18  }
19
20  const { record } = await request.json();
21  const contentId = record.id;
22  const contentText = record.content_text || '';
23
24  // Run OpenAI Moderation and custom rules in parallel
25  const [aiResult, ruleResult] = await Promise.all([
26    moderateWithOpenAI(contentText),
27    moderateWithRules(contentText),
28  ]);
29
30  // Combine scores (max of AI score and rule severity)
31  const combinedScore = Math.max(aiResult.score, ruleResult.score);
32  const categories = [...aiResult.categories, ...ruleResult.matchedRules];
33
34  // Update content item with scores
35  await supabase.from('content_items').update({
36    auto_score: combinedScore,
37    auto_categories: categories,
38    status: combinedScore < AUTO_APPROVE_THRESHOLD ? 'approved' : 'pending',
39  }).eq('id', contentId);
40
41  return NextResponse.json({
42    contentId,
43    score: combinedScore,
44    status: combinedScore < AUTO_APPROVE_THRESHOLD ? 'auto_approved' : 'queued_for_review',
45  });
46}
47
48async function moderateWithOpenAI(text: string) {
49  if (!text.trim()) return { score: 0, categories: [] };
50
51  const response = await fetch('https://api.openai.com/v1/moderations', {
52    method: 'POST',
53    headers: {
54      'Content-Type': 'application/json',
55      Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
56    },
57    body: JSON.stringify({ input: text }),
58  });
59
60  const data = await response.json();
61  const result = data.results[0];
62
63  // Get the highest category score
64  const scores = Object.values(result.category_scores) as number[];
65  const maxScore = Math.max(...scores);
66
67  // Get flagged categories
68  const flaggedCategories = Object.entries(result.categories)
69    .filter(([, flagged]) => flagged)
70    .map(([category]) => category);
71
72  return { score: maxScore, categories: flaggedCategories };
73}
74
75async function moderateWithRules(text: string) {
76  const { data: rules } = await supabase
77    .from('moderation_rules')
78    .select('*')
79    .eq('is_active', true);
80
81  const matchedRules: string[] = [];
82  let maxScore = 0;
83
84  for (const rule of rules ?? []) {
85    try {
86      const regex = new RegExp(rule.pattern, 'gi');
87      if (regex.test(text)) {
88        matchedRules.push(rule.name);
89        maxScore = Math.max(maxScore, rule.action === 'auto_reject' ? 1.0 : 0.7);
90      }
91    } catch {
92      // Skip invalid regex patterns
93    }
94  }
95
96  return { score: maxScore, matchedRules };
97}

1// app/moderation/page.tsx
2import { createClient } from '@/lib/supabase/server';
3import { Badge } from '@/components/ui/badge';
4import { Button } from '@/components/ui/button';
5import { Card, CardContent } from '@/components/ui/card';
6import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
7import { ModerationActions } from './moderation-actions';
8import { Shield, AlertTriangle, CheckCircle, XCircle } from 'lucide-react';
9
10function getSeverityColor(score: number): string {
11  if (score >= 0.8) return 'bg-red-100 text-red-800';
12  if (score >= 0.5) return 'bg-orange-100 text-orange-800';
13  if (score >= 0.3) return 'bg-yellow-100 text-yellow-800';
14  return 'bg-green-100 text-green-800';
15}
16
17export default async function ModerationPage() {
18  const supabase = await createClient();
19
20  const { data: items } = await supabase
21    .from('content_items')
22    .select('*')
23    .in('status', ['pending', 'escalated'])
24    .order('auto_score', { ascending: false })
25    .order('created_at', { ascending: true });
26
27  const { data: stats } = await supabase
28    .from('content_items')
29    .select('status')
30    .in('status', ['pending', 'escalated']);
31
32  const pending = stats?.filter(s => s.status === 'pending').length ?? 0;
33  const escalated = stats?.filter(s => s.status === 'escalated').length ?? 0;
34  const allItems = items ?? [];
35
36  return (
37    <div className="max-w-5xl mx-auto p-6 space-y-6">
38      <div className="flex items-center justify-between">
39        <div className="flex items-center gap-3">
40          <Shield className="h-8 w-8 text-primary" />
41          <div>
42            <h1 className="text-2xl font-bold">Content Moderation</h1>
43            <p className="text-sm text-muted-foreground">
44              {pending} pending · {escalated} escalated
45            </p>
46          </div>
47        </div>
48      </div>
49
50      <Tabs defaultValue="all">
51        <TabsList>
52          <TabsTrigger value="all">All ({allItems.length})</TabsTrigger>
53          <TabsTrigger value="pending">Pending ({pending})</TabsTrigger>
54          <TabsTrigger value="escalated">Escalated ({escalated})</TabsTrigger>
55        </TabsList>
56
57        {['all', 'pending', 'escalated'].map((tab) => (
58          <TabsContent key={tab} value={tab} className="space-y-3">
59            {allItems
60              .filter(item => tab === 'all' || item.status === tab)
61              .map((item) => (
62                <Card key={item.id}>
63                  <CardContent className="pt-4">
64                    <div className="flex items-start gap-4">
65                      <div className="flex-1 min-w-0">
66                        <div className="flex items-center gap-2 mb-2">
67                          <Badge variant="outline">{item.content_type}</Badge>
68                          <Badge className={getSeverityColor(item.auto_score ?? 0)}>
69                            Score: {(item.auto_score ?? 0).toFixed(2)}
70                          </Badge>
71                          {item.auto_categories?.map((cat: string) => (
72                            <Badge key={cat} variant="secondary" className="text-xs">
73                              {cat}
74                            </Badge>
75                          ))}
76                        </div>
77                        <p className="text-sm whitespace-pre-wrap line-clamp-3">
78                          {item.content_text}
79                        </p>
80                        {item.content_url && (
81                          <p className="text-xs text-muted-foreground mt-1">
82                            Media: {item.content_url}
83                          </p>
84                        )}
85                        <p className="text-xs text-muted-foreground mt-2">
86                          Submitted {new Date(item.created_at).toLocaleString()}
87                        </p>
88                      </div>
89                      <ModerationActions contentId={item.id} />
90                    </div>
91                  </CardContent>
92                </Card>
93              ))}
94          </TabsContent>
95        ))}
96      </Tabs>
97    </div>
98  );
99}
100
101// app/moderation/moderation-actions.tsx
102'use client';
103
104import { useState } from 'react';
105import { Button } from '@/components/ui/button';
106import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog';
107import { Textarea } from '@/components/ui/textarea';
108import { CheckCircle, XCircle, AlertTriangle, Ban } from 'lucide-react';
109import { moderateContent } from '@/app/actions/moderate';
110
111export function ModerationActions({ contentId }: { contentId: string }) {
112  const [reason, setReason] = useState('');
113  const [loading, setLoading] = useState(false);
114
115  async function handleAction(action: string) {
116    setLoading(true);
117    await moderateContent(contentId, action, reason);
118    setReason('');
119    setLoading(false);
120  }
121
122  return (
123    <div className="flex flex-col gap-1">
124      <Button size="sm" variant="outline" className="text-green-600"
125        onClick={() => handleAction('approve')} disabled={loading}>
126        <CheckCircle className="h-3 w-3 mr-1" /> Approve
127      </Button>
128      <Button size="sm" variant="outline" className="text-red-600"
129        onClick={() => handleAction('reject')} disabled={loading}>
130        <XCircle className="h-3 w-3 mr-1" /> Reject
131      </Button>
132      <Button size="sm" variant="outline" className="text-orange-600"
133        onClick={() => handleAction('escalate')} disabled={loading}>
134        <AlertTriangle className="h-3 w-3 mr-1" /> Escalate
135      </Button>
136      <Dialog>
137        <DialogTrigger asChild>
138          <Button size="sm" variant="outline" className="text-red-800">
139            <Ban className="h-3 w-3 mr-1" /> Warn
140          </Button>
141        </DialogTrigger>
142        <DialogContent>
143          <DialogHeader><DialogTitle>Issue Warning</DialogTitle></DialogHeader>
144          <Textarea placeholder="Reason for warning..." value={reason}
145            onChange={e => setReason(e.target.value)} />
146          <Button onClick={() => handleAction('warn')} disabled={loading}>
147            {loading ? 'Sending...' : 'Send Warning'}
148          </Button>
149        </DialogContent>
150      </Dialog>
151    </div>
152  );
153}

1// app/actions/moderate.ts
2'use server';
3
4import { createClient } from '@/lib/supabase/server';
5import { revalidatePath } from 'next/cache';
6
7export async function moderateContent(
8  contentId: string,
9  action: string,
10  reason?: string
11) {
12  const supabase = await createClient();
13  const { data: { user } } = await supabase.auth.getUser();
14  if (!user) throw new Error('Not authenticated');
15
16  // Map action to content status
17  const statusMap: Record<string, string> = {
18    approve: 'approved',
19    reject: 'rejected',
20    escalate: 'escalated',
21    warn: 'rejected',
22    ban: 'rejected',
23  };
24
25  // Update content item status
26  await supabase
27    .from('content_items')
28    .update({ status: statusMap[action] })
29    .eq('id', contentId);
30
31  // Create audit trail entry
32  // This INSERT triggers the handle_moderation_action function
33  // which automatically handles strikes and bans
34  await supabase.from('moderation_actions').insert({
35    content_id: contentId,
36    moderator_id: user.id,
37    action,
38    reason: reason || null,
39  });
40
41  revalidatePath('/moderation');
42  return { success: true };
43}
44
45// app/moderation/history/page.tsx
46import { createClient } from '@/lib/supabase/server';
47import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table';
48import { Badge } from '@/components/ui/badge';
49
50const actionColors: Record<string, string> = {
51  approve: 'bg-green-100 text-green-800',
52  reject: 'bg-red-100 text-red-800',
53  escalate: 'bg-orange-100 text-orange-800',
54  warn: 'bg-yellow-100 text-yellow-800',
55  ban: 'bg-red-200 text-red-900',
56};
57
58export default async function HistoryPage() {
59  const supabase = await createClient();
60
61  const { data: actions } = await supabase
62    .from('moderation_actions')
63    .select('*, content_items(content_type, content_text)')
64    .order('created_at', { ascending: false })
65    .limit(100);
66
67  return (
68    <div className="max-w-5xl mx-auto p-6">
69      <h1 className="text-2xl font-bold mb-6">Moderation History</h1>
70      <Table>
71        <TableHeader>
72          <TableRow>
73            <TableHead>Date</TableHead>
74            <TableHead>Content Type</TableHead>
75            <TableHead>Action</TableHead>
76            <TableHead>Reason</TableHead>
77            <TableHead>Content Preview</TableHead>
78          </TableRow>
79        </TableHeader>
80        <TableBody>
81          {actions?.map((action) => (
82            <TableRow key={action.id}>
83              <TableCell className="text-sm">
84                {new Date(action.created_at).toLocaleString()}
85              </TableCell>
86              <TableCell>
87                <Badge variant="outline">{action.content_items?.content_type}</Badge>
88              </TableCell>
89              <TableCell>
90                <Badge className={actionColors[action.action]}>{action.action}</Badge>
91              </TableCell>
92              <TableCell className="text-sm text-muted-foreground max-w-[200px] truncate">
93                {action.reason || '—'}
94              </TableCell>
95              <TableCell className="text-sm max-w-[300px] truncate">
96                {action.content_items?.content_text}
97              </TableCell>
98            </TableRow>
99          ))}
100        </TableBody>
101      </Table>
102    </div>
103  );
104}

1// Supabase Database Webhook configuration:
2// 1. Go to Supabase Dashboard > Database > Webhooks
3// 2. Create new webhook:
4//    - Name: auto-moderate-content
5//    - Table: content_items
6//    - Events: INSERT
7//    - Type: HTTP Request
8//    - Method: POST
9//    - URL: https://your-app.vercel.app/api/moderate/auto
10//    - Headers: Authorization: Bearer YOUR_WEBHOOK_SECRET
11
12// Environment variables for V0's Vars tab:
13// OPENAI_API_KEY=sk-...
14// MODERATION_WEBHOOK_SECRET=your-random-secret-string
15// NEXT_PUBLIC_SUPABASE_URL=https://xxx.supabase.co
16// NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJ...
17// SUPABASE_SERVICE_ROLE_KEY=eyJ...
18
19// app/moderation/rules/page.tsx
20'use client';
21
22import { useState, useEffect } from 'react';
23import { createClient } from '@/lib/supabase/client';
24import { Button } from '@/components/ui/button';
25import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
26import { Input } from '@/components/ui/input';
27import { Switch } from '@/components/ui/switch';
28import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table';
29import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
30import { Plus, Trash2 } from 'lucide-react';
31
32type Rule = {
33  id: string;
34  name: string;
35  pattern: string;
36  action: string;
37  is_active: boolean;
38};
39
40export default function RulesPage() {
41  const supabase = createClient();
42  const [rules, setRules] = useState<Rule[]>([]);
43  const [newName, setNewName] = useState('');
44  const [newPattern, setNewPattern] = useState('');
45  const [newAction, setNewAction] = useState('flag');
46
47  useEffect(() => {
48    supabase.from('moderation_rules').select('*').order('created_at').then(({ data }) => {
49      if (data) setRules(data);
50    });
51  }, []);
52
53  async function addRule() {
54    if (!newName || !newPattern) return;
55    // Validate regex
56    try { new RegExp(newPattern); } catch { return; }
57
58    const { data } = await supabase.from('moderation_rules').insert({
59      name: newName, pattern: newPattern, action: newAction,
60    }).select().single();
61
62    if (data) {
63      setRules([...rules, data]);
64      setNewName(''); setNewPattern('');
65    }
66  }
67
68  async function toggleRule(id: string, is_active: boolean) {
69    await supabase.from('moderation_rules').update({ is_active }).eq('id', id);
70    setRules(rules.map(r => r.id === id ? { ...r, is_active } : r));
71  }
72
73  async function deleteRule(id: string) {
74    await supabase.from('moderation_rules').delete().eq('id', id);
75    setRules(rules.filter(r => r.id !== id));
76  }
77
78  return (
79    <div className="max-w-4xl mx-auto p-6 space-y-6">
80      <h1 className="text-2xl font-bold">Moderation Rules</h1>
81
82      <Card>
83        <CardHeader><CardTitle className="text-lg">Add Rule</CardTitle></CardHeader>
84        <CardContent className="flex gap-2">
85          <Input placeholder="Rule name" value={newName} onChange={e => setNewName(e.target.value)} />
86          <Input placeholder="Regex pattern" value={newPattern} onChange={e => setNewPattern(e.target.value)} className="font-mono" />
87          <Select value={newAction} onValueChange={setNewAction}>
88            <SelectTrigger className="w-32"><SelectValue /></SelectTrigger>
89            <SelectContent>
90              <SelectItem value="flag">Flag</SelectItem>
91              <SelectItem value="auto_reject">Auto-reject</SelectItem>
92            </SelectContent>
93          </Select>
94          <Button onClick={addRule}><Plus className="h-4 w-4" /></Button>
95        </CardContent>
96      </Card>
97
98      <Table>
99        <TableHeader>
100          <TableRow>
101            <TableHead>Active</TableHead>
102            <TableHead>Name</TableHead>
103            <TableHead>Pattern</TableHead>
104            <TableHead>Action</TableHead>
105            <TableHead></TableHead>
106          </TableRow>
107        </TableHeader>
108        <TableBody>
109          {rules.map(rule => (
110            <TableRow key={rule.id}>
111              <TableCell>
112                <Switch checked={rule.is_active} onCheckedChange={v => toggleRule(rule.id, v)} />
113              </TableCell>
114              <TableCell className="font-medium">{rule.name}</TableCell>
115              <TableCell className="font-mono text-sm">{rule.pattern}</TableCell>
116              <TableCell>{rule.action}</TableCell>
117              <TableCell>
118                <Button variant="ghost" size="icon" onClick={() => deleteRule(rule.id)}>
119                  <Trash2 className="h-4 w-4" />
120                </Button>
121              </TableCell>
122            </TableRow>
123          ))}
124        </TableBody>
125      </Table>
126    </div>
127  );
128}