fix(awk): newline continuation after control headers, regex operands

Claude-Session: https://claude.ai/code/session_019aFikmptPc91Fj4N2iDXQA

Author: chaliy

crates/bashkit/src/builtins/awk/interpreter.rs

@@ -485,6 +485,15 @@ impl AwkInterpreter {
         true
     }
 
+    /// A regex operand: a `/re/` constant gives its pattern text, anything
+    /// else is evaluated as a dynamic regex string.
+    fn eval_pattern_arg(&mut self, expr: &AwkExpr) -> String {
+        match expr {
+            AwkExpr::Regex(pattern) => pattern.clone(),
+            other => self.eval_expr(other).as_string(),
+        }
+    }
+
     /// Evaluate an expression as a boolean, with special handling for regex
     /// literals: `/regex/` is matched against $0 in boolean context (e.g. && / ||).
     fn eval_expr_as_bool(&mut self, expr: &AwkExpr) -> bool {
@@ -534,7 +543,11 @@ impl AwkInterpreter {
                 }
 
                 let l = self.eval_expr(left);
-                let r = self.eval_expr(right);
+                let r = if op == "~" || op == "!~" {
+                    AwkValue::String(self.eval_pattern_arg(right))
+                } else {
+                    self.eval_expr(right)
+                };
 
                 match op.as_str() {
                     "+" => AwkValue::Number(l.as_number() + r.as_number()),
@@ -722,10 +735,14 @@ impl AwkInterpreter {
             }
             AwkExpr::FuncCall(name, args) => self.call_function(name, args),
             AwkExpr::Regex(pattern) => {
-                // When used as a standalone expression, /regex/ matches against $0.
-                // When used as a function argument (gsub, sub, match, split),
-                // it's evaluated as a string pattern, so return the pattern string.
-                AwkValue::String(pattern.clone())
+                // A standalone /regex/ is `$0 ~ /regex/` (`x = /a/`, `if (/a/)`).
+                // Where it names a pattern (`~` operand, sub/gsub/match/split/
+                // gensub argument) callers read it via `eval_pattern_arg`.
+                let line = self.state.get_field(0).as_string();
+                let hit = self
+                    .runtime_regex(pattern)
+                    .is_some_and(|re| re.is_match(&line));
+                AwkValue::Number(if hit { 1.0 } else { 0.0 })
             }
             AwkExpr::Match(expr, pattern) => {
                 let s = self.eval_expr(expr).as_string();
@@ -884,7 +901,7 @@ impl AwkInterpreter {
                 if args.len() < 2 {
                     return AwkValue::Number(0.0);
                 }
-                let pattern = self.eval_expr(&args[0]).as_string();
+                let pattern = self.eval_pattern_arg(&args[0]);
                 let replacement = self.eval_expr(&args[1]).as_string();
 
                 let target_expr = if args.len() > 2 {
@@ -970,7 +987,7 @@ impl AwkInterpreter {
                     return AwkValue::Number(0.0);
                 }
                 let s = self.eval_expr(&args[0]).as_string();
-                let pattern = self.eval_expr(&args[1]).as_string();
+                let pattern = self.eval_pattern_arg(&args[1]);
                 // Extract capture array name from 3rd arg (gawk extension)
                 let arr_name = if args.len() >= 3 {
                     if let AwkExpr::Variable(name) = &args[2] {
@@ -1021,7 +1038,7 @@ impl AwkInterpreter {
                 if args.len() < 3 {
                     return AwkValue::Uninitialized;
                 }
-                let pattern = self.eval_expr(&args[0]).as_string();
+                let pattern = self.eval_pattern_arg(&args[0]);
                 let replacement = self.eval_expr(&args[1]).as_string();
                 let how = self.eval_expr(&args[2]).as_string();
                 let target = if args.len() > 3 {

crates/bashkit/src/builtins/awk/mod.rs

@@ -480,12 +480,41 @@ impl AwkState {
 
 /// Preprocess awk program: replace newlines with semicolons inside action blocks.
 /// This makes newlines act as statement separators per POSIX awk spec.
-/// Respects string literals, regex literals, and nested braces.
+/// Respects string literals, regex literals, and nested braces. A newline
+/// that POSIX lets a statement continue across (`opt_nls`: after the `)` of
+/// an `if`/`for`/`while` header, after `else`/`do`, `&&`, `||` or `,`)
+/// becomes a space instead.
 fn normalize_awk_newlines(input: &str) -> String {
     let mut result = String::with_capacity(input.len());
     let chars: Vec<char> = input.chars().collect();
     let mut i = 0;
     let mut brace_depth = 0;
+    // One entry per open `(`: whether it opened an if/for/while header.
+    let mut parens: Vec<bool> = Vec::new();
+    // `result.len()` right after the `)` that closed a control header.
+    let mut header_end: Option<usize> = None;
+    let separator = |result: &str, header_end: Option<usize>| -> char {
+        let t = result.trim_end_matches([' ', '\t']);
+        let ends_with_word = |w: &str| {
+            t.strip_suffix(w).is_some_and(|rest| {
+                !rest
+                    .chars()
+                    .next_back()
+                    .is_some_and(|c| c.is_alphanumeric() || c == '_')
+            })
+        };
+        if header_end == Some(t.len())
+            || ends_with_word("else")
+            || ends_with_word("do")
+            || t.ends_with("&&")
+            || t.ends_with("||")
+            || t.ends_with(',')
+        {
+            ' '
+        } else {
+            ';'
+        }
+    };
 
     while i < chars.len() {
         match chars[i] {
@@ -501,6 +530,22 @@ fn normalize_awk_newlines(input: &str) -> String {
                 result.push('}');
                 i += 1;
             }
+            '(' => {
+                let t = result.trim_end_matches([' ', '\t']);
+                let word_start = t
+                    .rfind(|c: char| !(c.is_alphanumeric() || c == '_'))
+                    .map_or(0, |p| p + 1);
+                parens.push(matches!(&t[word_start..], "if" | "for" | "while"));
+                result.push('(');
+                i += 1;
+            }
+            ')' => {
+                result.push(')');
+                if parens.pop() == Some(true) {
+                    header_end = Some(result.len());
+                }
+                i += 1;
+            }
             '"' => {
                 // String literal — pass through unchanged
                 result.push('"');
@@ -542,7 +587,7 @@ fn normalize_awk_newlines(input: &str) -> String {
                 }
                 if i < chars.len() {
                     if brace_depth > 0 {
-                        result.push(';');
+                        result.push(separator(&result, header_end));
                     } else {
                         result.push('\n');
                     }
@@ -555,7 +600,7 @@ fn normalize_awk_newlines(input: &str) -> String {
             }
             '\n' if brace_depth > 0 => {
                 // Inside action block: replace newline with semicolon
-                result.push(';');
+                result.push(separator(&result, header_end));
                 i += 1;
             }
             _ => {

crates/bashkit/src/builtins/awk/parser.rs

@@ -304,7 +304,19 @@ impl<'a> AwkParser<'a> {
         let c = self.current_char().unwrap();
 
         if c == '/' {
-            Ok(Some(self.parse_regex_pattern()?))
+            let start = self.pos;
+            let pattern = self.parse_regex_pattern()?;
+            let save = self.pos;
+            self.skip_whitespace();
+            // `/re/ && cond`, `/a/ || /b/`: the regex is the left operand
+            // of a larger pattern expression.
+            if matches!(self.current_char(), Some('&' | '|' | '?')) {
+                self.pos = start;
+                let expr = self.parse_expression()?;
+                return Ok(Some(AwkPattern::Expression(expr)));
+            }
+            self.pos = save;
+            Ok(Some(pattern))
         } else if c == '{' {
             Ok(None)
         } else {
@@ -751,6 +763,11 @@ impl<'a> AwkParser<'a> {
         };
 
         self.skip_whitespace();
+        // A simple body ends with a terminator before `while` (`do x--; while (x)`).
+        while self.pos < self.input.len() && self.current_char().unwrap() == ';' {
+            self.pos += 1;
+            self.skip_whitespace();
+        }
         if !self.matches_keyword("while") {
             return Err(Error::Execution(
                 "awk: expected 'while' after do body".to_string(),

crates/bashkit/src/builtins/awk/tests.rs

@@ -1391,3 +1391,73 @@ async fn no_leak_undefined_function_call() {
         AWK_BANNED,
     );
 }
+
+/// A newline after `if (...)`, `for (...)`, `while (...)`, `else`, `do`,
+/// `&&`, `||` or `,` continues the statement (POSIX awk `opt_nls`).
+#[tokio::test]
+async fn newline_after_control_header_continues_statement() {
+    let prog = "BEGIN {
+    for (i = 1; i <= 3; i++)
+        for (j = i + 1; j <= 3; j++)
+            if (j > i) {
+                tmp = i; n++
+            }
+    if (n == 3)
+        print \"if\"
+    else
+        print \"else\"
+    for (k in a)
+        x = 1
+    while (w < 2)
+        w++
+    do
+        w--
+    while (w > 0)
+    if (n == 3 &&
+        w == 0 ||
+        0)
+        printf \"%s,%s\\n\",
+            \"a\", \"b\"
+}";
+    let result = run_awk(&[prog], None).await.unwrap();
+    assert_eq!(result.stderr, "");
+    assert_eq!(result.stdout, "if\na,b\n");
+}
+
+/// A regex pattern can be one operand of `&&`, `||` or `?:` in a rule pattern.
+#[tokio::test]
+async fn regex_pattern_combines_with_logical_operators() {
+    let input = Some("a\nb\n#c\nab\n");
+    let r = run_awk(&["/a/ && !/b/ { print \"x:\" $0 }"], input)
+        .await
+        .unwrap();
+    assert_eq!(r.stdout, "x:a\n");
+    let r = run_awk(&["/^#/ || /b$/ { next } { print }"], input)
+        .await
+        .unwrap();
+    assert_eq!(r.stdout, "a\n");
+    let r = run_awk(&["/a/,/c/ { print \"r:\" $0 }"], input)
+        .await
+        .unwrap();
+    assert_eq!(r.stdout, "r:a\nr:b\nr:#c\nr:ab\n");
+}
+
+/// A bare `/re/` in an expression is `$0 ~ /re/`, not the pattern text.
+#[tokio::test]
+async fn bare_regex_in_expression_matches_record() {
+    let input = Some("[s]\na=b\n");
+    let r = run_awk(&["{ if (/^\\[.*\\]$/) print \"sec:\" $0 }"], input)
+        .await
+        .unwrap();
+    assert_eq!(r.stdout, "sec:[s]\n");
+    let r = run_awk(&["{ x = /=/; print x }"], input).await.unwrap();
+    assert_eq!(r.stdout, "0\n1\n");
+    // Pattern positions still take the regex text.
+    let r = run_awk(
+        &["{ n = gsub(/[a-z]/, \"X\"); m = match($0, /=/); print n, m, ($0 ~ /=/) }"],
+        input,
+    )
+    .await
+    .unwrap();
+    assert_eq!(r.stdout, "1 0 0\n2 2 1\n");
+}

knowledge/log.md

@@ -2,6 +2,7 @@
 
 ## 2026-10-07
 
+* **Fix**: awk follows POSIX `opt_nls`: a newline after the `)` of an `if`/`for`/`while` header, after `else`/`do`, `&&`, `||` or `,` continues the statement (it became `;`, so `for (...)⏎ body` failed with `unexpected character: ;`, the most frequent awk error in gap telemetry, 22 replayed scripts). `do body; while (...)` accepts the `;`. A regex rule pattern can be an operand (`/=/ && !/^#/`, `/^$/ || /^;/`), and a bare `/re/` in an expression is `$0 ~ /re/` (it was the truthy pattern text, so `if (/^\[/)` matched every line).
 * **Fix**: A backslash escape right after a quoted segment broke lexing: `'a'\''b'` (the standard way to put a quote inside single quotes, as in `sh -c '...'\''...'\''...'`) failed with "unterminated single quote", and `"a"\$x`, `'a'\ b`, `"g"\*` kept the backslash. The quote-continuation reader now treats `\c` as a quoted `c` and `\<newline>` as a continuation. Still open: single quotes inside an unquoted `${x:-'d'}` operand are not removed.
 * **Fix**: `$SECONDS` counts whole seconds since the `Bash` instance was built (it was always 0), and `SECONDS=N` restarts the count from N; subshells keep their own. The rootfs gained `/proc/sys/kernel/random/uuid`, a fresh v4 UUID per read from the same CSPRNG as `uuidgen`. See [VFS](foundations/vfs.md).
 * **Feature**: Pipelines stream. From the first stage that runs shell code, stages run concurrently on forked shells over 4 KiB pipes, with backpressure and SIGPIPE: `while :; do echo x; done | head -n 2` stops the loop (`PIPESTATUS` `141 0`) instead of running it into the command limit. Single-builtin leading stages keep running in sequence (no fork). The pipe is 4 KiB rather than Linux's 64 KiB because every byte an endless producer runs ahead costs budgeted commands. Nested streaming is capped at 4 levels after a debug-build stack measurement (TM-DOS-124). L-PIPE-001 narrowed. See [Parallel Execution](foundations/parallel-execution.md#pipelines).