Stream command results as dots in real time

src/cli/index.ts

@@ -6,9 +6,10 @@ import { program } from "commander"
 import ansis from "ansis"
 
 import { parse, parseSetup, type Command, type ShoutFile } from "../parse.ts"
-import { runFile, cleanupTmpDir } from "../run.ts"
+import { runFile, cleanupTmpDir, type CommandResult } from "../run.ts"
 import { evaluateFile, formatFailure, formatSummary } from "../format.ts"
 import type { TestResult } from "../format.ts"
+import { matchOutput } from "../match.ts"
 import { parseDuration } from "../duration.ts"
 import { rewriteFile } from "../update.ts"
 
@@ -138,6 +139,26 @@ program
       }
       const merged: ShoutFile = { ...parsed, commands: [...setupCommands, ...parsed.commands, ...teardownCommands] }
 
+      const setupLen = setupCommands.length
+      const userLen = parsed.commands.length
+      const printDot = (result: CommandResult) => {
+        const { command, actual, exitCode } = result
+        const outputMatches = matchOutput(command.expected, actual)
+        let exitCodeMismatch = false
+        if (command.exitCode === null) {
+          exitCodeMismatch = exitCode !== 0
+        } else if (command.exitCode === "*") {
+          exitCodeMismatch = exitCode === 0
+        } else {
+          exitCodeMismatch = exitCode !== command.exitCode
+        }
+        if (outputMatches && !exitCodeMismatch) {
+          process.stdout.write(ansis.green("."))
+        } else {
+          process.stdout.write(ansis.red("F"))
+        }
+      }
+
       const fileResult = await runFile(merged, {
         cleanEnv: opts.cleanEnv ?? false,
         pathDirs: opts.path,
@@ -148,6 +169,11 @@ program
         onCommand: opts.verbose
           ? (cmd) => process.stderr.write(ansis.dim(`  $ ${cmd.command}\n`))
           : undefined,
+        onCommandResult: (index, result) => {
+          if (index >= setupLen && index < setupLen + userLen) {
+            printDot(result)
+          }
+        },
       })
 
       // Check setup commands for failures
@@ -211,24 +237,16 @@ program
       return testResult
     }
 
-    const printDots = (r: TestResult) => {
+    const printErrorDot = (r: TestResult) => {
       if (r.error) {
         process.stdout.write(ansis.red("F"))
-        return
-      }
-      const passed = r.commandCount - r.failures.length
-      for (let i = 0; i < passed; i++) {
-        process.stdout.write(ansis.green("."))
-      }
-      for (let i = 0; i < r.failures.length; i++) {
-        process.stdout.write(ansis.red("F"))
       }
     }
 
     if (opts.parallel) {
       const promises = files.map(async f => {
         const r = await runOne(f, nextPort++)
-        printDots(r)
+        printErrorDot(r)
         return r
       })
       results.push(...await Promise.all(promises))
@@ -236,7 +254,7 @@ program
     } else {
       for (const filePath of files) {
         const r = await runOne(filePath, nextPort++)
-        printDots(r)
+        printErrorDot(r)
         results.push(r)
       }
       process.stdout.write("\n")

src/run.ts

@@ -25,7 +25,9 @@ type RunOptions = {
   sourceDir?: string
   projectDir?: string
   timeout: number
+  verbose?: boolean
   onCommand?: (cmd: Command) => void
+  onCommandResult?: (index: number, result: CommandResult) => void
 }
 
 function killTree(pid: number): void {
@@ -192,7 +194,7 @@ export async function runFile(
   }
 
   const sentinel = SENTINEL_PREFIX
-  const verbose = options.verbose && !!options.onCommand
+  const verbose = !!(options.verbose && options.onCommand)
   const script = buildScript(file.commands, sentinel, verbose)
 
   const env: Record<string, string> = options.cleanEnv
@@ -236,7 +238,21 @@ export async function runFile(
 
     const totalTimeout = options.timeout * file.commands.length
     const lastSentinelSuffix = `_${file.commands.length - 1}__`
-    const stdout = await readUntilSentinel(proc.stdout, sentinel, lastSentinelSuffix, totalTimeout)
+    const onSentinel = options.onCommandResult
+      ? (index: number, exitCode: number, rawOutput: string) => {
+          let lines = rawOutput.split("\n")
+          if (lines.length > 0 && lines[0] === "") lines.shift()
+          lines = trimTrailingEmpty(lines)
+          if (lines.length === 1 && lines[0] === "") lines = []
+          const result: CommandResult = {
+            command: file.commands[index]!,
+            actual: lines.map(stripAnsi),
+            exitCode,
+          }
+          options.onCommandResult!(index, result)
+        }
+      : undefined
+    const stdout = await readUntilSentinel(proc.stdout, sentinel, lastSentinelSuffix, totalTimeout, onSentinel)
     if (!verbose) {
       await readWithTimeout(proc.stderr, 1000).catch(() => "")
     }
@@ -272,10 +288,13 @@ async function readUntilSentinel(
   sentinelPrefix: string,
   sentinelSuffix: string,
   timeoutMs: number,
+  onSentinel?: (index: number, exitCode: number, outputBefore: string) => void,
 ): Promise<string> {
   const reader = stream.getReader()
   const decoder = new TextDecoder()
   let accumulated = ""
+  let sentinelsReported = 0
+  let lastSentinelEnd = 0
 
   let timerId: ReturnType<typeof setTimeout>
   const timeout = new Promise<never>((_, reject) =>
@@ -288,6 +307,25 @@ async function readUntilSentinel(
       if (done) break
       if (value) {
         accumulated += decoder.decode(value, { stream: true })
+
+        // Detect new sentinels incrementally for streaming results
+        if (onSentinel) {
+          const regex = new RegExp(`${sentinelPrefix}(\\d+)_(\\d+)__`, "g")
+          regex.lastIndex = lastSentinelEnd
+          let match
+          while ((match = regex.exec(accumulated)) !== null) {
+            const idx = parseInt(match[2]!, 10)
+            if (idx >= sentinelsReported) {
+              const exitCode = parseInt(match[1]!, 10)
+              const output = accumulated.slice(lastSentinelEnd, match.index)
+              onSentinel(idx, exitCode, output)
+              sentinelsReported = idx + 1
+              lastSentinelEnd = match.index + match[0].length
+              if (accumulated[lastSentinelEnd] === "\n") lastSentinelEnd++
+            }
+          }
+        }
+
         // Check if the last sentinel has appeared (prefix + exitcode + suffix)
         const prefixIdx = accumulated.lastIndexOf(sentinelPrefix)
         if (prefixIdx !== -1 && accumulated.indexOf(sentinelSuffix, prefixIdx) !== -1) break