From 70008d16b9080e403765daca15bffec86789d050 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Tue, 10 Mar 2026 09:32:06 -0700 Subject: [PATCH 1/5] Add @env, @setup directives and --port-from flag --- src/cli/index.ts | 48 +++++++++++++++++++++++++++++++++++------ src/index.ts | 2 +- src/parse.test.ts | 43 ++++++++++++++++++++++++++++++++++++ src/parse.ts | 23 +++++++++++++++++++- src/run.ts | 5 +++++ test/env.shout | 5 +++++ test/setup-shared.shout | 1 + test/setup-user.shout | 4 ++++ 8 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 test/env.shout create mode 100644 test/setup-shared.shout create mode 100644 test/setup-user.shout diff --git a/src/cli/index.ts b/src/cli/index.ts index 735b737..a357e30 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,11 +1,11 @@ #!/usr/bin/env bun import { readdir, readFile, writeFile } from "node:fs/promises" -import { resolve, relative } from "node:path" +import { resolve, relative, dirname } from "node:path" import { program } from "commander" import ansis from "ansis" -import { parse } from "../parse.ts" +import { parse, type ShoutFile } from "../parse.ts" import { runFile, cleanupTmpDir } from "../run.ts" import { evaluateFile, formatFailure, formatSummary } from "../format.ts" import type { TestResult } from "../format.ts" @@ -53,6 +53,7 @@ program .option("--path ", "Prepend to PATH (repeatable)", (val: string, acc: string[]) => [...acc, val]) .option("--timeout ", "Per-command timeout", "10s") .option("-v, --verbose", "Print each command as it runs") + .option("--port-from ", "Auto-assign $PORT starting from ") .option("--parallel", "Run files in parallel") .option("--example", "Print an example .shout file and exit") .action(async (fileArgs: string[], opts) => { @@ -88,14 +89,40 @@ $ true const start = performance.now() const results: TestResult[] = [] const cwd = process.cwd() + const portFrom = opts.portFrom ? parseInt(opts.portFrom, 10) : undefined + let nextPort = portFrom ?? 0 - const runOne = async (filePath: string) => { + const runOne = async (filePath: string, port?: number) => { const content = await readFile(filePath, "utf-8") const parsed = parse(relative(cwd, filePath), content) - const fileResult = await runFile(parsed, { + // Collect env vars: --port-from, then @env directives + const envVars: Record = {} + if (port !== undefined) envVars["PORT"] = String(port) + for (const d of parsed.directives) { + if (d.type === "env") envVars[d.key] = d.value + } + + // Resolve @setup directives: prepend setup commands + let setupCount = 0 + let merged: ShoutFile = parsed + const setupDirectives = parsed.directives.filter(d => d.type === "setup") + if (setupDirectives.length > 0) { + const setupCommands = [] + for (const d of setupDirectives) { + const setupPath = resolve(dirname(filePath), d.path) + const setupContent = await readFile(setupPath, "utf-8") + const setupParsed = parse(relative(cwd, setupPath), setupContent) + setupCommands.push(...setupParsed.commands) + } + setupCount = setupCommands.length + merged = { ...parsed, commands: [...setupCommands, ...parsed.commands] } + } + + const fileResult = await runFile(merged, { cleanEnv: opts.cleanEnv ?? false, pathDirs: opts.path, + envVars: Object.keys(envVars).length > 0 ? envVars : undefined, timeout: timeoutMs, verbose: opts.verbose ?? false, onCommand: opts.verbose @@ -110,7 +137,9 @@ $ true ) if (opts.update && fileResult.results.length > 0) { - const updated = rewriteFile(parsed, fileResult.results, content) + // Only rewrite with the file's own results, not setup results + const fileOwnResults = fileResult.results.slice(setupCount) + const updated = rewriteFile(parsed, fileOwnResults, content) if (updated !== content) { await writeFile(filePath, updated) } @@ -140,7 +169,11 @@ $ true } if (opts.parallel) { - const all = await Promise.all(files.map(runOne)) + const tasks = files.map(f => { + const port = portFrom !== undefined ? nextPort++ : undefined + return runOne(f, port) + }) + const all = await Promise.all(tasks) for (const r of all) { printDots(r) results.push(r) @@ -148,7 +181,8 @@ $ true process.stdout.write("\n") } else { for (const filePath of files) { - const r = await runOne(filePath) + const port = portFrom !== undefined ? nextPort++ : undefined + const r = await runOne(filePath, port) printDots(r) results.push(r) } diff --git a/src/index.ts b/src/index.ts index 107ee68..29281a8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -export type { Command, ShoutFile } from "./parse.ts" +export type { Command, Directive, ShoutFile } from "./parse.ts" export type { CommandResult, FileResult } from "./run.ts" export type { DiffLine } from "./match.ts" export type { TestResult } from "./format.ts" diff --git a/src/parse.test.ts b/src/parse.test.ts index 87712ec..8af05ba 100644 --- a/src/parse.test.ts +++ b/src/parse.test.ts @@ -66,4 +66,47 @@ describe("parse", () => { const b = parse("test.shout", "$ echo hi\nhi") expect(a.commands[0]!.expected).toEqual(b.commands[0]!.expected) }) + + test("@env directive", () => { + const result = parse("test.shout", "@env PORT=3000\n$ echo $PORT\n3000\n") + expect(result.directives).toEqual([ + { type: "env", key: "PORT", value: "3000", line: 1 }, + ]) + expect(result.commands).toHaveLength(1) + }) + + test("@env with value containing =", () => { + const result = parse("test.shout", "@env FOO=bar=baz\n$ echo $FOO\n") + expect(result.directives[0]).toEqual( + { type: "env", key: "FOO", value: "bar=baz", line: 1 }, + ) + }) + + test("@setup directive", () => { + const result = parse("test.shout", "@setup shared/setup.shout\n$ echo hi\nhi\n") + expect(result.directives).toEqual([ + { type: "setup", path: "shared/setup.shout", line: 1 }, + ]) + expect(result.commands).toHaveLength(1) + }) + + test("multiple directives", () => { + const content = "@setup setup.shout\n@env PORT=3000\n@env NODE_ENV=test\n\n$ echo hi\nhi\n" + const result = parse("test.shout", content) + expect(result.directives).toHaveLength(3) + expect(result.directives[0]!.type).toBe("setup") + expect(result.directives[1]).toEqual({ type: "env", key: "PORT", value: "3000", line: 2 }) + expect(result.directives[2]).toEqual({ type: "env", key: "NODE_ENV", value: "test", line: 3 }) + }) + + test("@ lines after first command are expected output", () => { + const result = parse("test.shout", "$ cat config\n@env PORT=3000\n") + expect(result.directives).toEqual([]) + expect(result.commands[0]!.expected).toEqual(["@env PORT=3000"]) + }) + + test("no directives returns empty array", () => { + const result = parse("test.shout", "$ echo hi\nhi\n") + expect(result.directives).toEqual([]) + }) }) diff --git a/src/parse.ts b/src/parse.ts index d204ca4..bd62914 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -6,9 +6,14 @@ export type Command = { exitCode: number | "*" | null } +export type Directive = + | { type: "setup"; path: string; line: number } + | { type: "env"; key: string; value: string; line: number } + export type ShoutFile = { path: string commands: Command[] + directives: Directive[] } function stripComment(line: string): string { @@ -58,12 +63,28 @@ export function parse(path: string, content: string): ShoutFile { } const commands: Command[] = [] + const directives: Directive[] = [] let current: Command | null = null + let seenCommand = false for (let i = 0; i < rawLines.length; i++) { const line = rawLines[i]! + if (!seenCommand && line.startsWith("@")) { + if (line.startsWith("@setup ")) { + directives.push({ type: "setup", path: line.slice(7).trim(), line: i + 1 }) + } else if (line.startsWith("@env ")) { + const rest = line.slice(5).trim() + const eq = rest.indexOf("=") + if (eq > 0) { + directives.push({ type: "env", key: rest.slice(0, eq), value: rest.slice(eq + 1), line: i + 1 }) + } + } + continue + } + if (line.startsWith("$ ")) { + seenCommand = true if (current) { const trimmed = trimTrailingEmpty(current.expected) const { lines: expectedLines, exitCode } = parseExitCode(trimmed) @@ -92,5 +113,5 @@ export function parse(path: string, content: string): ShoutFile { commands.push(current) } - return { path, commands } + return { path, commands, directives } } diff --git a/src/run.ts b/src/run.ts index 001ac4d..0233c7f 100644 --- a/src/run.ts +++ b/src/run.ts @@ -20,6 +20,7 @@ export type FileResult = { type RunOptions = { cleanEnv: boolean pathDirs?: string[] + envVars?: Record timeout: number verbose: boolean onCommand?: (cmd: Command) => void @@ -128,6 +129,10 @@ export async function runFile( env["HOME"] = tmpDir env["CUE_DIR"] = tmpDir + if (options.envVars) { + Object.assign(env, options.envVars) + } + if (options.pathDirs?.length) { env["PATH"] = options.pathDirs.join(":") + ":" + (env["PATH"] ?? "") } diff --git a/test/env.shout b/test/env.shout new file mode 100644 index 0000000..245d02c --- /dev/null +++ b/test/env.shout @@ -0,0 +1,5 @@ +@env GREETING=hello +@env TARGET=world + +$ echo "$GREETING $TARGET" +hello world diff --git a/test/setup-shared.shout b/test/setup-shared.shout new file mode 100644 index 0000000..7ad3eba --- /dev/null +++ b/test/setup-shared.shout @@ -0,0 +1 @@ +$ export READY=yes diff --git a/test/setup-user.shout b/test/setup-user.shout new file mode 100644 index 0000000..023c5c2 --- /dev/null +++ b/test/setup-user.shout @@ -0,0 +1,4 @@ +@setup setup-shared.shout + +$ echo $READY +yes From 86eba1a6242b547f45768d5478e7502cd8eeb515 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Tue, 10 Mar 2026 09:45:43 -0700 Subject: [PATCH 2/5] docs: add @env/@setup directives to CLAUDE.md --- CLAUDE.md | 4 +++- src/cli/index.ts | 34 ++++++++++++++++------------------ src/parse.test.ts | 6 ++++++ src/parse.ts | 2 ++ 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9bf54dc..20ab1e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ Transcript-based shell integration test runner. Bun + TypeScript. - `bun test` — run unit tests - `bunx tsc --noEmit` — type check -- `bun run src/cli/index.ts [files...]` — run shout CLI +- `bun run src/cli/index.ts [files...]` — run shout CLI (`--port-from ` auto-assigns `$PORT`) ## Architecture @@ -28,6 +28,8 @@ Transcript-based shell integration test runner. Bun + TypeScript. - `[N]` on last line of expected output = assert exit code N - `[*]` = assert any non-zero exit code; default expects 0 - `#` after a command = comment (stripped); `#` in expected output is literal +- `@env KEY=VALUE` before first command = set environment variable +- `@setup path.shout` before first command = prepend commands (and `@env`) from another file - Each file runs in a fresh temp dir with a single `/bin/sh` session ## Style diff --git a/src/cli/index.ts b/src/cli/index.ts index a357e30..292dbee 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -5,7 +5,7 @@ import { resolve, relative, dirname } from "node:path" import { program } from "commander" import ansis from "ansis" -import { parse, type ShoutFile } from "../parse.ts" +import { parse, type Command, type ShoutFile } from "../parse.ts" import { runFile, cleanupTmpDir } from "../run.ts" import { evaluateFile, formatFailure, formatSummary } from "../format.ts" import type { TestResult } from "../format.ts" @@ -103,21 +103,19 @@ $ true if (d.type === "env") envVars[d.key] = d.value } - // Resolve @setup directives: prepend setup commands - let setupCount = 0 - let merged: ShoutFile = parsed - const setupDirectives = parsed.directives.filter(d => d.type === "setup") - if (setupDirectives.length > 0) { - const setupCommands = [] - for (const d of setupDirectives) { - const setupPath = resolve(dirname(filePath), d.path) - const setupContent = await readFile(setupPath, "utf-8") - const setupParsed = parse(relative(cwd, setupPath), setupContent) - setupCommands.push(...setupParsed.commands) + // Resolve @setup directives: prepend setup commands, collect their @env + const setupCommands: Command[] = [] + for (const d of parsed.directives) { + if (d.type !== "setup") continue + const setupPath = resolve(dirname(filePath), d.path) + const setupContent = await readFile(setupPath, "utf-8") + const setupParsed = parse(relative(cwd, setupPath), setupContent) + for (const sd of setupParsed.directives) { + if (sd.type === "env") envVars[sd.key] = sd.value } - setupCount = setupCommands.length - merged = { ...parsed, commands: [...setupCommands, ...parsed.commands] } + setupCommands.push(...setupParsed.commands) } + const merged: ShoutFile = { ...parsed, commands: [...setupCommands, ...parsed.commands] } const fileResult = await runFile(merged, { cleanEnv: opts.cleanEnv ?? false, @@ -130,15 +128,15 @@ $ true : undefined, }) + const fileOwnResults = fileResult.results.slice(setupCommands.length) + const testResult = evaluateFile( parsed.path, - fileResult.results, + fileOwnResults, fileResult.error, ) - if (opts.update && fileResult.results.length > 0) { - // Only rewrite with the file's own results, not setup results - const fileOwnResults = fileResult.results.slice(setupCount) + if (opts.update && fileOwnResults.length > 0) { const updated = rewriteFile(parsed, fileOwnResults, content) if (updated !== content) { await writeFile(filePath, updated) diff --git a/src/parse.test.ts b/src/parse.test.ts index 8af05ba..70a28f7 100644 --- a/src/parse.test.ts +++ b/src/parse.test.ts @@ -109,4 +109,10 @@ describe("parse", () => { const result = parse("test.shout", "$ echo hi\nhi\n") expect(result.directives).toEqual([]) }) + + test("unknown directive throws", () => { + expect(() => parse("test.shout", "@evn PORT=3000\n$ echo hi\n")).toThrow( + "test.shout:1: unknown directive: @evn PORT=3000", + ) + }) }) diff --git a/src/parse.ts b/src/parse.ts index bd62914..4bd07a4 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -79,6 +79,8 @@ export function parse(path: string, content: string): ShoutFile { if (eq > 0) { directives.push({ type: "env", key: rest.slice(0, eq), value: rest.slice(eq + 1), line: i + 1 }) } + } else { + throw new Error(`${path}:${i + 1}: unknown directive: ${line}`) } continue } From e97be11a0c6922a3dd8ca39142246cf970b200ef Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Tue, 10 Mar 2026 10:03:17 -0700 Subject: [PATCH 3/5] Fix setup env precedence and validate directives --- src/cli/index.ts | 61 +++++++++++++++++++++++++++++++----------------- src/parse.ts | 5 ++-- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/cli/index.ts b/src/cli/index.ts index 292dbee..d25b178 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -90,37 +90,46 @@ $ true const results: TestResult[] = [] const cwd = process.cwd() const portFrom = opts.portFrom ? parseInt(opts.portFrom, 10) : undefined + if (portFrom !== undefined && Number.isNaN(portFrom)) { + console.error("--port-from must be an integer") + process.exit(1) + } let nextPort = portFrom ?? 0 - const runOne = async (filePath: string, port?: number) => { + const runOne = async (filePath: string) => { const content = await readFile(filePath, "utf-8") const parsed = parse(relative(cwd, filePath), content) - // Collect env vars: --port-from, then @env directives + // Resolve directives in a single pass. Setup @env is collected separately + // so that the user file's @env always takes precedence. const envVars: Record = {} - if (port !== undefined) envVars["PORT"] = String(port) - for (const d of parsed.directives) { - if (d.type === "env") envVars[d.key] = d.value - } - - // Resolve @setup directives: prepend setup commands, collect their @env + if (portFrom !== undefined) envVars["PORT"] = String(nextPort++) + const setupEnvVars: Record = {} + const userEnvVars: Record = {} const setupCommands: Command[] = [] for (const d of parsed.directives) { - if (d.type !== "setup") continue - const setupPath = resolve(dirname(filePath), d.path) - const setupContent = await readFile(setupPath, "utf-8") - const setupParsed = parse(relative(cwd, setupPath), setupContent) - for (const sd of setupParsed.directives) { - if (sd.type === "env") envVars[sd.key] = sd.value + if (d.type === "setup") { + const setupPath = resolve(dirname(filePath), d.path) + const setupContent = await readFile(setupPath, "utf-8") + const setupParsed = parse(relative(cwd, setupPath), setupContent) + for (const sd of setupParsed.directives) { + if (sd.type === "setup") { + throw new Error(`${relative(cwd, setupPath)}: @setup not allowed in setup files`) + } + if (sd.type === "env") setupEnvVars[sd.key] = sd.value + } + setupCommands.push(...setupParsed.commands) + } else if (d.type === "env") { + userEnvVars[d.key] = d.value } - setupCommands.push(...setupParsed.commands) } + Object.assign(envVars, setupEnvVars, userEnvVars) const merged: ShoutFile = { ...parsed, commands: [...setupCommands, ...parsed.commands] } const fileResult = await runFile(merged, { cleanEnv: opts.cleanEnv ?? false, pathDirs: opts.path, - envVars: Object.keys(envVars).length > 0 ? envVars : undefined, + envVars, timeout: timeoutMs, verbose: opts.verbose ?? false, onCommand: opts.verbose @@ -128,6 +137,18 @@ $ true : undefined, }) + // Check setup commands for failures + for (let i = 0; i < setupCommands.length; i++) { + const r = fileResult.results[i] + if (r && r.exitCode !== 0) { + return evaluateFile( + parsed.path, + [], + `setup command failed (exit ${r.exitCode}): $ ${setupCommands[i]!.command}`, + ) + } + } + const fileOwnResults = fileResult.results.slice(setupCommands.length) const testResult = evaluateFile( @@ -167,10 +188,7 @@ $ true } if (opts.parallel) { - const tasks = files.map(f => { - const port = portFrom !== undefined ? nextPort++ : undefined - return runOne(f, port) - }) + const tasks = files.map(f => runOne(f)) const all = await Promise.all(tasks) for (const r of all) { printDots(r) @@ -179,8 +197,7 @@ $ true process.stdout.write("\n") } else { for (const filePath of files) { - const port = portFrom !== undefined ? nextPort++ : undefined - const r = await runOne(filePath, port) + const r = await runOne(filePath) printDots(r) results.push(r) } diff --git a/src/parse.ts b/src/parse.ts index 4bd07a4..615204a 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -76,9 +76,10 @@ export function parse(path: string, content: string): ShoutFile { } else if (line.startsWith("@env ")) { const rest = line.slice(5).trim() const eq = rest.indexOf("=") - if (eq > 0) { - directives.push({ type: "env", key: rest.slice(0, eq), value: rest.slice(eq + 1), line: i + 1 }) + if (eq <= 0) { + throw new Error(`${path}:${i + 1}: malformed @env directive (expected KEY=VALUE): ${line}`) } + directives.push({ type: "env", key: rest.slice(0, eq), value: rest.slice(eq + 1), line: i + 1 }) } else { throw new Error(`${path}:${i + 1}: unknown directive: ${line}`) } From 0a1100aaf752ae2531bea2bf9a837f8841c3a63b Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Tue, 10 Mar 2026 10:08:53 -0700 Subject: [PATCH 4/5] Fix port assignment to respect user-defined PORT vars --- src/cli/index.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cli/index.ts b/src/cli/index.ts index d25b178..f73bf30 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -94,16 +94,15 @@ $ true console.error("--port-from must be an integer") process.exit(1) } - let nextPort = portFrom ?? 0 + let nextPort = portFrom - const runOne = async (filePath: string) => { + const runOne = async (filePath: string, port: number | undefined) => { const content = await readFile(filePath, "utf-8") const parsed = parse(relative(cwd, filePath), content) // Resolve directives in a single pass. Setup @env is collected separately // so that the user file's @env always takes precedence. const envVars: Record = {} - if (portFrom !== undefined) envVars["PORT"] = String(nextPort++) const setupEnvVars: Record = {} const userEnvVars: Record = {} const setupCommands: Command[] = [] @@ -124,6 +123,9 @@ $ true } } Object.assign(envVars, setupEnvVars, userEnvVars) + if (port !== undefined && !("PORT" in userEnvVars) && !("PORT" in setupEnvVars)) { + envVars["PORT"] = String(port) + } const merged: ShoutFile = { ...parsed, commands: [...setupCommands, ...parsed.commands] } const fileResult = await runFile(merged, { @@ -188,8 +190,7 @@ $ true } if (opts.parallel) { - const tasks = files.map(f => runOne(f)) - const all = await Promise.all(tasks) + const all = await Promise.all(files.map(f => runOne(f, nextPort !== undefined ? nextPort++ : undefined))) for (const r of all) { printDots(r) results.push(r) @@ -197,7 +198,7 @@ $ true process.stdout.write("\n") } else { for (const filePath of files) { - const r = await runOne(filePath) + const r = await runOne(filePath, nextPort !== undefined ? nextPort++ : undefined) printDots(r) results.push(r) } From aeef6041ec39af34ee999f040f5e0849d9793c26 Mon Sep 17 00:00:00 2001 From: Chris Wanstrath Date: Tue, 10 Mar 2026 10:20:23 -0700 Subject: [PATCH 5/5] Add exit code expectations and fix @setup validation --- src/cli/index.ts | 15 +++++++++++++-- src/parse.ts | 6 +++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/cli/index.ts b/src/cli/index.ts index f73bf30..eff10c6 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -142,11 +142,22 @@ $ true // Check setup commands for failures for (let i = 0; i < setupCommands.length; i++) { const r = fileResult.results[i] - if (r && r.exitCode !== 0) { + const expected = setupCommands[i]!.exitCode + const ok = expected === null + ? r?.exitCode === 0 + : expected === "*" + ? r?.exitCode !== 0 + : r?.exitCode === expected + if (!ok) { + if (opts.keep) { + process.stderr.write(`${fileResult.tmpDir}\n`) + } else { + await cleanupTmpDir(fileResult.tmpDir) + } return evaluateFile( parsed.path, [], - `setup command failed (exit ${r.exitCode}): $ ${setupCommands[i]!.command}`, + `setup command failed (exit ${r?.exitCode ?? "?"}): $ ${setupCommands[i]!.command}`, ) } } diff --git a/src/parse.ts b/src/parse.ts index 615204a..94cb20c 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -72,7 +72,11 @@ export function parse(path: string, content: string): ShoutFile { if (!seenCommand && line.startsWith("@")) { if (line.startsWith("@setup ")) { - directives.push({ type: "setup", path: line.slice(7).trim(), line: i + 1 }) + const setupPath = line.slice(7).trim() + if (!setupPath) { + throw new Error(`${path}:${i + 1}: @setup requires a file path`) + } + directives.push({ type: "setup", path: setupPath, line: i + 1 }) } else if (line.startsWith("@env ")) { const rest = line.slice(5).trim() const eq = rest.indexOf("=")