From be026007599ec05a8f191cb7207b4e6448a01a82 Mon Sep 17 00:00:00 2001 From: Marco Sadjadi Date: Sun, 31 May 2026 19:52:16 +0200 Subject: [PATCH] feat(security): block credentials from reaching the LLM via prompt secret scan Prompts were sent to the model with no secret scan, so a pasted API key would leak to the LLM. Added findSecretInPrompt in @bmm/types (tight provider-key patterns: Anthropic/OpenAI/GitHub/AWS/Google/Slack/Stripe/JWT/private-key) shared by both sides. The web wizard blocks before sending with a clear message; the API preview and preview-stream endpoints reject with secret_in_prompt as the hard guarantee. Credential VALUES already never touched the model - they are entered in the separate encrypted step 2; this closes the remaining leak path where a user pastes a key into the prompt itself. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/api/src/routes/servers.ts | 21 +++++++++++++++ apps/web/app/(dashboard)/servers/new/page.tsx | 11 ++++++++ packages/types/src/index.ts | 27 +++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/apps/api/src/routes/servers.ts b/apps/api/src/routes/servers.ts index 58cc81c..b839bda 100644 --- a/apps/api/src/routes/servers.ts +++ b/apps/api/src/routes/servers.ts @@ -28,6 +28,7 @@ import { IterateServerInput, PreviewInput, type SpecEdit, + findSecretInPrompt, } from '@bmm/types'; import type { FastifyInstance } from 'fastify'; import { z } from 'zod'; @@ -62,6 +63,16 @@ export async function serverRoutes(app: FastifyInstance): Promise { if (!parsed.success) { return reply.code(400).send({ error: 'invalid_input', issues: parsed.error.flatten() }); } + // Never let a credential reach the LLM. Reject prompts that contain a + // real-looking key/token before the model call. (Values belong in the + // separate encrypted credential fields, not the prompt.) + const leakedSecret = findSecretInPrompt(parsed.data.prompt); + if (leakedSecret) { + return reply.code(400).send({ + error: 'secret_in_prompt', + detail: `Your prompt looks like it contains ${leakedSecret}. Remove it — API keys must never go in the prompt (it is sent to the AI model). You will add credentials in their own encrypted fields after the spec is generated.`, + }); + } const billing = await getOrgBilling(user.orgId); if (billing.suspended) { @@ -190,6 +201,16 @@ export async function serverRoutes(app: FastifyInstance): Promise { if (!parsed.success) { return reply.code(400).send({ error: 'invalid_input', issues: parsed.error.flatten() }); } + // Never let a credential reach the LLM. Reject prompts that contain a + // real-looking key/token before the model call. (Values belong in the + // separate encrypted credential fields, not the prompt.) + const leakedSecret = findSecretInPrompt(parsed.data.prompt); + if (leakedSecret) { + return reply.code(400).send({ + error: 'secret_in_prompt', + detail: `Your prompt looks like it contains ${leakedSecret}. Remove it — API keys must never go in the prompt (it is sent to the AI model). You will add credentials in their own encrypted fields after the spec is generated.`, + }); + } const billing = await getOrgBilling(user.orgId); if (billing.suspended) { diff --git a/apps/web/app/(dashboard)/servers/new/page.tsx b/apps/web/app/(dashboard)/servers/new/page.tsx index 6702af4..40901ba 100644 --- a/apps/web/app/(dashboard)/servers/new/page.tsx +++ b/apps/web/app/(dashboard)/servers/new/page.tsx @@ -6,6 +6,7 @@ import { InstallSnippets } from '@/components/install-snippets'; import { StreamingLogs } from '@/components/streaming-logs'; import { Button } from '@/components/ui/button'; import { apiFetch, apiSseStream } from '@/lib/api'; +import { findSecretInPrompt } from '@bmm/types'; import { Loader2, RotateCcw, X } from 'lucide-react'; import Link from 'next/link'; import { useRouter, useSearchParams } from 'next/navigation'; @@ -240,6 +241,16 @@ function NewServerPageInner() { setError('Name and slug are required.'); return; } + // Keep credentials out of the model: block a prompt that contains a real + // key/token before it is ever sent. Credentials go in the encrypted fields + // in the next step, never in the prompt. + const leaked = findSecretInPrompt(prompt); + if (leaked) { + setError( + `Looks like your prompt contains ${leaked}. Remove it — API keys must never go in the prompt (it is sent to the AI). You'll add credentials in their own encrypted fields in the next step.`, + ); + return; + } setStep('analyzing'); // Streaming preview: pipes Anthropic's token deltas back as SSE. Cloudflare's diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 511bc71..0a3282b 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -212,3 +212,30 @@ export const InstallTarget = z.enum([ 'raw-url', ]); export type InstallTarget = z.infer; + +// ---- Prompt secret guard ---- +// Unambiguous provider key/token shapes. Used to reject a prompt that contains +// a real credential BEFORE it is sent to the LLM — keys must never reach the +// model. Kept tight (provider prefixes / structural markers) so normal prompts +// don't trip it. Shared by the API preview endpoints and the web wizard. +const PROMPT_SECRET_PATTERNS: ReadonlyArray<{ name: string; re: RegExp }> = [ + { name: 'an Anthropic key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}/ }, + { name: 'an OpenAI project key', re: /\bsk-proj-[A-Za-z0-9_-]{20,}/ }, + { name: 'an OpenAI key', re: /\bsk-[A-Za-z0-9]{32,}\b/ }, + { name: 'a GitHub token', re: /\bgh[posru]_[A-Za-z0-9]{30,}\b/ }, + { name: 'an AWS access key', re: /\bAKIA[0-9A-Z]{16}\b/ }, + { name: 'a Google API key', re: /\bAIza[0-9A-Za-z_-]{35}\b/ }, + { name: 'a Slack token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}/ }, + { name: 'a Stripe secret key', re: /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{20,}/ }, + { name: 'a Stripe publishable key', re: /\bpk_(?:live|test)_[A-Za-z0-9]{20,}/ }, + { name: 'a JWT / bearer token', re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/ }, + { name: 'a private key block', re: /-----BEGIN (?:RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----/ }, +]; + +/** First secret-like token found in the text, else null. */ +export function findSecretInPrompt(text: string): string | null { + for (const { name, re } of PROMPT_SECRET_PATTERNS) { + if (re.test(text)) return name; + } + return null; +}