diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 5cb2a33d5c..e5a17aed8c 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -325,6 +325,8 @@ jobs: - name: "pinact" id: "pinact" continue-on-error: true + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" uses: *just with: recipe: "pinact" diff --git a/Cargo.lock b/Cargo.lock index f40b119244..15b8f3787a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1169,7 +1169,7 @@ dependencies = [ [[package]] name = "dataplane" -version = "0.26.0" +version = "0.25.2" dependencies = [ "afpacket", "arrayvec", @@ -1217,7 +1217,7 @@ dependencies = [ [[package]] name = "dataplane-acl" -version = "0.26.0" +version = "0.25.2" dependencies = [ "arrayvec", "bolero", @@ -1233,7 +1233,7 @@ dependencies = [ [[package]] name = "dataplane-acl-filter" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bolero", "dataplane-acl", @@ -1259,7 +1259,7 @@ dependencies = [ [[package]] name = "dataplane-args" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bytecheck", "clap", @@ -1282,7 +1282,7 @@ dependencies = [ [[package]] name = "dataplane-cli" -version = "0.26.0" +version = "0.25.2" dependencies = [ "clap", "colored", @@ -1298,7 +1298,7 @@ dependencies = [ [[package]] name = "dataplane-common" -version = "0.26.0" +version = "0.25.2" dependencies = [ "arc-swap", "dataplane-concurrency", @@ -1307,7 +1307,7 @@ dependencies = [ [[package]] name = "dataplane-concurrency" -version = "0.26.0" +version = "0.25.2" dependencies = [ "arc-swap", "bolero", @@ -1320,7 +1320,7 @@ dependencies = [ [[package]] name = "dataplane-concurrency-macros" -version = "0.26.0" +version = "0.25.2" dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", @@ -1330,7 +1330,7 @@ dependencies = [ [[package]] name = "dataplane-config" -version = "0.26.0" +version = "0.25.2" dependencies = [ "arc-swap", "bolero", @@ -1356,7 +1356,7 @@ dependencies = [ [[package]] name = "dataplane-dpdk" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bolero", "dataplane-concurrency", @@ -1375,7 +1375,7 @@ dependencies = [ [[package]] name = "dataplane-dpdk-sys" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bindgen", "dataplane-dpdk-sysroot-helper", @@ -1384,11 +1384,11 @@ dependencies = [ [[package]] name = "dataplane-dpdk-sysroot-helper" -version = "0.26.0" +version = "0.25.2" [[package]] name = "dataplane-dpdk-test-macros" -version = "0.26.0" +version = "0.25.2" dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", @@ -1398,18 +1398,18 @@ dependencies = [ [[package]] name = "dataplane-errno" -version = "0.26.0" +version = "0.25.2" dependencies = [ "thiserror", ] [[package]] name = "dataplane-fixed-size" -version = "0.26.0" +version = "0.25.2" [[package]] name = "dataplane-flow-entry" -version = "0.26.0" +version = "0.25.2" dependencies = [ "ahash", "bolero", @@ -1430,7 +1430,7 @@ dependencies = [ [[package]] name = "dataplane-flow-filter" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bolero", "dataplane-acl", @@ -1452,7 +1452,7 @@ dependencies = [ [[package]] name = "dataplane-hardware" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bolero", "bytecheck", @@ -1478,7 +1478,7 @@ dependencies = [ [[package]] name = "dataplane-id" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bolero", "rkyv", @@ -1488,7 +1488,7 @@ dependencies = [ [[package]] name = "dataplane-init" -version = "0.26.0" +version = "0.25.2" dependencies = [ "dataplane-dpdk-sysroot-helper", "dataplane-hardware", @@ -1505,7 +1505,7 @@ dependencies = [ [[package]] name = "dataplane-interface-manager" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bolero", "caps", @@ -1533,7 +1533,7 @@ dependencies = [ [[package]] name = "dataplane-k8s-intf" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bolero", "dataplane-concurrency", @@ -1558,7 +1558,7 @@ dependencies = [ [[package]] name = "dataplane-k8s-less" -version = "0.26.0" +version = "0.25.2" dependencies = [ "dataplane-k8s-intf", "dataplane-tracectl", @@ -1570,7 +1570,7 @@ dependencies = [ [[package]] name = "dataplane-left-right-tlcache" -version = "0.26.0" +version = "0.25.2" dependencies = [ "ahash", "dataplane-concurrency", @@ -1581,7 +1581,7 @@ dependencies = [ [[package]] name = "dataplane-lifecycle" -version = "0.26.0" +version = "0.25.2" dependencies = [ "dataplane-concurrency", "tokio", @@ -1591,11 +1591,11 @@ dependencies = [ [[package]] name = "dataplane-lookup" -version = "0.26.0" +version = "0.25.2" [[package]] name = "dataplane-lpm" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bnum", "bolero", @@ -1611,7 +1611,7 @@ dependencies = [ [[package]] name = "dataplane-match-action" -version = "0.26.0" +version = "0.25.2" dependencies = [ "arrayvec", "bolero", @@ -1621,7 +1621,7 @@ dependencies = [ [[package]] name = "dataplane-match-action-derive" -version = "0.26.0" +version = "0.25.2" dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", @@ -1631,7 +1631,7 @@ dependencies = [ [[package]] name = "dataplane-mgmt" -version = "0.26.0" +version = "0.25.2" dependencies = [ "bolero", "bytes", @@ -1680,7 +1680,7 @@ dependencies = [ [[package]] name = "dataplane-nat" -version = "0.26.0" +version = "0.25.2" dependencies = [ "ahash", "arc-swap", @@ -1714,7 +1714,7 @@ dependencies = [ [[package]] name = "dataplane-net" -version = "0.26.0" +version = "0.25.2" dependencies = [ "ahash", "arrayvec", @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "dataplane-pipeline" -version = "0.26.0" +version = "0.25.2" dependencies = [ "dataplane-concurrency", "dataplane-id", @@ -1759,11 +1759,11 @@ dependencies = [ [[package]] name = "dataplane-rekon" -version = "0.26.0" +version = "0.25.2" [[package]] name = "dataplane-routing" -version = "0.26.0" +version = "0.25.2" dependencies = [ "ahash", "anyhow", @@ -1808,7 +1808,7 @@ dependencies = [ [[package]] name = "dataplane-stats" -version = "0.26.0" +version = "0.25.2" dependencies = [ "arrayvec", "bolero", @@ -1833,7 +1833,7 @@ dependencies = [ [[package]] name = "dataplane-sysfs" -version = "0.26.0" +version = "0.25.2" dependencies = [ "dataplane-concurrency", "n-vm", @@ -1845,7 +1845,7 @@ dependencies = [ [[package]] name = "dataplane-test-utils" -version = "0.26.0" +version = "0.25.2" dependencies = [ "caps", "nix 0.31.3", @@ -1856,7 +1856,7 @@ dependencies = [ [[package]] name = "dataplane-tracectl" -version = "0.26.0" +version = "0.25.2" dependencies = [ "color-eyre", "dataplane-common", @@ -1873,7 +1873,7 @@ dependencies = [ [[package]] name = "dataplane-validator" -version = "0.26.0" +version = "0.25.2" dependencies = [ "dataplane-config", "dataplane-k8s-intf", @@ -1883,7 +1883,7 @@ dependencies = [ [[package]] name = "dataplane-vpcmap" -version = "0.26.0" +version = "0.25.2" dependencies = [ "ahash", "bolero", diff --git a/Cargo.toml b/Cargo.toml index 264c0eab38..7db5d56b0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ members = [ resolver = "3" [workspace.package] -version = "0.26.0" +version = "0.25.2" edition = "2024" license = "Apache-2.0" publish = false diff --git a/default.nix b/default.nix index 301dba5408..4f8964ec67 100644 --- a/default.nix +++ b/default.nix @@ -159,10 +159,15 @@ let cargo-deny cargo-depgraph cargo-edit + cargo-expand cargo-llvm-cov + cargo-mutants cargo-nextest + cargo-show-asm commitlint-rs + deno direnv + duvet gateway-crd gettext jq diff --git a/justfile b/justfile index 6eed433ff8..b96bf7b534 100644 --- a/justfile +++ b/justfile @@ -44,6 +44,8 @@ profile := "debug" # sanitizer to use (address/thread/safe-stack/cfi/"") sanitize := "" +bolero_coverage_test_time_ms := env("BOLERO_COVERAGE_TEST_TIME_MS", "15000") + # comma-separated list of cargo features to enable (e.g. "shuttle") features := "" @@ -740,6 +742,7 @@ doctest package="" *args: (build (if package == "" { "doctests.all" } else { "do [script] coverage *args: {{ _just_debuggable_ }} + export BOLERO_RANDOM_TEST_TIME_MS="{{ bolero_coverage_test_time_ms }}" export LLVM_COV="$(pwd)/devroot/bin/llvm-cov" export LLVM_PROFDATA="$(pwd)/devroot/bin/llvm-profdata" declare -r out="./target/nextest/coverage" @@ -747,9 +750,46 @@ coverage *args: cargo llvm-cov --no-report --branch nextest {{ args }} mkdir -p "${out}" cargo llvm-cov report --branch --html --output-dir="${out}" + cargo llvm-cov report --branch --lcov --output-path="${out}/lcov.info" cargo llvm-cov report --branch --codecov --output-path="${out}/codecov.json" cargo llvm-cov report --branch --summary-only +[script] +duvet *args: + {{ _just_debuggable_ }} + duvet report {{ args }} + +[script] +duvet-check: + {{ _just_debuggable_ }} + for input in .duvet/config.toml .duvet/snapshot.txt; do + if [ ! -f "${input}" ]; then + echo "error: ${input} is missing; this check has nothing to compare and cannot pass" >&2 + exit 1 + fi + done + duvet report + if ! git diff --quiet -- .duvet/snapshot.txt; then + echo "error: .duvet/snapshot.txt is stale; run \`just duvet\` and commit the result" >&2 + git --no-pager diff -- .duvet/snapshot.txt >&2 + exit 1 + fi + +[script] +mutants *args: + {{ _just_debuggable_ }} + cargo mutants --test-tool nextest {{ args }} + +[script] +spec-interlock *args: + {{ _just_debuggable_ }} + ./scripts/spec-interlock.ts {{ args }} + +[script] +duvet-summary *args: + {{ _just_debuggable_ }} + ./scripts/duvet-summary.ts {{ args }} | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}" + # Use Nix-built archives so local and CI coverage report the same binaries. [script] coverage-archive package="tests.all" *args: diff --git a/net/src/headers/mod.rs b/net/src/headers/mod.rs index e715486a10..22fb5237ca 100644 --- a/net/src/headers/mod.rs +++ b/net/src/headers/mod.rs @@ -890,39 +890,7 @@ impl Headers { } } - /// The number of octets which follow the transport header according to the network header. - /// - /// A frame can carry more octets than its network header accounts for: ethernet pads short - /// frames out to 60 octets (`IEEE 802.3` clause 4.2.3.3), and some devices append trailers. - /// Those octets belong to no upper layer, so a caller holding a whole frame needs this to find - /// where the transport payload ends before computing a checksum over it. - /// - /// # Returns - /// - /// Returns `None` when the network or transport header is absent, or when the network header - /// claims fewer octets than the headers which follow it occupy -- a malformed packet, for which - /// no honest payload length exists. - pub(crate) fn transport_payload_len(&self) -> Option { - let ip_payload_len = match self.net.as_ref()? { - Net::Ipv4(ip) => usize::from(ip.0.payload_len().ok()?), - Net::Ipv6(ip) => usize::from(ip.0.payload_length), - }; - let after_net = self - .net_ext - .iter() - .map(|ext| usize::from(ext.size().get())) - .sum::() - + usize::from(self.transport.as_ref()?.size().get()); - ip_payload_len.checked_sub(after_net) - } - /// update the checksums of the headers - /// - /// `payload` must be exactly the octets which follow the transport header, and no more: it is - /// summed in full. A caller working from a frame buffer should bound it with - /// [`Headers::transport_payload_len`] first, as [`Packet::update_checksums`] does. - /// - /// [`Packet::update_checksums`]: crate::packet::Packet::update_checksums pub(crate) fn update_checksums(&mut self, payload: impl AsRef<[u8]>) { let is_vxlan = self.try_vxlan().is_some(); diff --git a/net/src/packet/mod.rs b/net/src/packet/mod.rs index d56025da4e..3cdee54bf3 100644 --- a/net/src/packet/mod.rs +++ b/net/src/packet/mod.rs @@ -326,22 +326,8 @@ impl Packet { } /// Update the network and transport checksums based on the current headers. - /// - /// Only the octets the network header vouches for are summed. A frame can hold more than - /// that: ethernet pads short frames out to 60 octets, and some devices append trailers. Those - /// octets are not payload, and including them corrupts the result -- the pseudo header length - /// comes out too large for TCP and `ICMPv6`, and any non-zero octet perturbs the sum for every - /// protocol. - /// - /// A buffer holding *less* than the network header claims is left as-is: the packet is - /// truncated, and no choice of payload yields a checksum the far end will accept. pub fn update_checksums(&mut self) -> &mut Self { - let payload = self.payload.as_ref(); - let payload = match self.headers.transport_payload_len() { - Some(len) if len <= payload.len() => &payload[..len], - _ => payload, - }; - self.headers.update_checksums(payload); + self.headers.update_checksums(&self.payload); self.meta_mut().set_checksum_refresh(false); self } @@ -902,142 +888,3 @@ mod qos_roundtrip_tests { } } } - -#[cfg(test)] -mod padding_tests { - use crate::buffer::TestBuffer; - use crate::checksum::Checksum; - use crate::headers::{TryHeaders, TryIcmp4, TryIp, TryTcp, TryUdp}; - use crate::packet::Packet; - use crate::tcp::TcpChecksumPayload; - use crate::udp::UdpChecksumPayload; - - /// The smallest frame ethernet will carry, per `IEEE 802.3` clause 4.2.3.3. - const MIN_ETHERNET_FRAME: usize = 60; - - /// An ethernet + IPv4 frame carrying `l4`, with the IPv4 total length describing `l4` exactly. - fn frame(protocol: u8, l4: &[u8]) -> Vec { - let mut frame = Vec::new(); - frame.extend_from_slice(&[0x02, 0, 0, 0, 0, 1]); // destination mac - frame.extend_from_slice(&[0x02, 0, 0, 0, 0, 2]); // source mac - frame.extend_from_slice(&[0x08, 0x00]); // ipv4 - frame.extend_from_slice(&[0x45, 0x00]); - #[allow(clippy::cast_possible_truncation)] // test input is small - frame.extend_from_slice(&((20 + l4.len()) as u16).to_be_bytes()); - frame.extend_from_slice(&[0x00, 0x01, 0x00, 0x00, 0x40, protocol, 0x00, 0x00]); - frame.extend_from_slice(&[192, 168, 0, 1]); // source ip - frame.extend_from_slice(&[192, 168, 0, 2]); // destination ip - frame.extend_from_slice(l4); - frame - } - - /// A bare TCP acknowledgement: no options, no payload, 20 octets. - fn tcp_ack() -> Vec { - let mut tcp = Vec::new(); - tcp.extend_from_slice(&1000_u16.to_be_bytes()); // source port - tcp.extend_from_slice(&2000_u16.to_be_bytes()); // destination port - tcp.extend_from_slice(&[0, 0, 0, 1]); // sequence number - tcp.extend_from_slice(&[0, 0, 0, 2]); // acknowledgement number - tcp.extend_from_slice(&[0x50, 0x10]); // data offset 5, ACK - tcp.extend_from_slice(&1024_u16.to_be_bytes()); // window - tcp.extend_from_slice(&[0, 0]); // checksum - tcp.extend_from_slice(&[0, 0]); // urgent pointer - tcp - } - - /// A UDP datagram header describing `payload`. - fn udp(payload: &[u8]) -> Vec { - let mut udp = Vec::new(); - udp.extend_from_slice(&1000_u16.to_be_bytes()); // source port - udp.extend_from_slice(&2000_u16.to_be_bytes()); // destination port - #[allow(clippy::cast_possible_truncation)] // test input is small - udp.extend_from_slice(&((8 + payload.len()) as u16).to_be_bytes()); - udp.extend_from_slice(&[0, 0]); // checksum - udp.extend_from_slice(payload); - udp - } - - /// An `ICMPv4` echo request, 8 octets. - fn icmp4_echo_request() -> Vec { - vec![8, 0, 0, 0, 0x00, 0x2a, 0x00, 0x01] - } - - fn parse(frame: &[u8]) -> Packet { - Packet::new(TestBuffer::from_raw_data(frame)).expect("frame does not parse") - } - - /// Pad `frame` out to the ethernet minimum with `filler`. - /// - /// `IEEE 802.3` requires the padding but does not constrain its content, so a peer may send - /// anything here. Historically it has sent the contents of uninitialized memory - /// (`CVE-2003-0001`). - fn pad(mut frame: Vec, filler: u8) -> Vec { - assert!(frame.len() < MIN_ETHERNET_FRAME, "frame needs no padding"); - frame.resize(MIN_ETHERNET_FRAME, filler); - frame - } - - #[test] - fn tcp_checksum_excludes_zeroed_ethernet_padding() { - let mut packet = parse(&pad(frame(6, &tcp_ack()), 0)); - packet.update_checksums(); - let net = packet.headers().try_ip().expect("no ip header").clone(); - packet - .headers() - .try_tcp() - .expect("no tcp header") - .validate_checksum(&TcpChecksumPayload::new(&net, &[])) - .expect("padding leaked into the tcp checksum"); - } - - #[test] - fn udp_checksum_excludes_non_zero_ethernet_padding() { - let mut packet = parse(&pad(frame(17, &udp(&[])), 0xab)); - packet.update_checksums(); - let net = packet.headers().try_ip().expect("no ip header").clone(); - packet - .headers() - .try_udp() - .expect("no udp header") - .validate_checksum(&UdpChecksumPayload::new(&net, &[])) - .expect("padding leaked into the udp checksum"); - } - - #[test] - fn icmp4_checksum_excludes_non_zero_ethernet_padding() { - let mut packet = parse(&pad(frame(1, &icmp4_echo_request()), 0xab)); - packet.update_checksums(); - packet - .headers() - .try_icmp4() - .expect("no icmp header") - .validate_checksum(&[]) - .expect("padding leaked into the icmp checksum"); - } - - #[test] - fn checksum_still_covers_a_real_payload() { - let payload: Vec = (0..32_u8).collect(); - let mut packet = parse(&frame(17, &udp(&payload))); - assert_eq!(packet.payload().as_ref(), payload.as_slice()); - packet.update_checksums(); - let net = packet.headers().try_ip().expect("no ip header").clone(); - packet - .headers() - .try_udp() - .expect("no udp header") - .validate_checksum(&UdpChecksumPayload::new(&net, &payload)) - .expect("payload dropped out of the udp checksum"); - } - - /// A payload shorter than the network header claims cannot be checksummed correctly by anyone. - /// All this asks is that we compute *something* rather than panic on the short slice. - #[test] - fn truncated_payload_does_not_panic() { - let payload: Vec = (0..32_u8).collect(); - let mut frame = frame(17, &udp(&payload)); - frame.truncate(frame.len() - 8); - let mut packet = parse(&frame); - packet.update_checksums(); - } -} diff --git a/net/src/packet/utils.rs b/net/src/packet/utils.rs index a6618d0550..6e333bce3a 100644 --- a/net/src/packet/utils.rs +++ b/net/src/packet/utils.rs @@ -147,6 +147,7 @@ impl Packet { /// Get the Ip protocol / next-header of an IPv4 / IPv6 [`Packet`] /// Returns None if the packet does not have an IP header + #[allow(missing_docs)] pub fn ip_proto(&self) -> Option { self.try_ip().map(|net| match net { Ipv4(ipv4) => NextHeader(ipv4.protocol()), diff --git a/nix/overlays/dataplane-dev.nix b/nix/overlays/dataplane-dev.nix index c8829018f2..16e84046d8 100644 --- a/nix/overlays/dataplane-dev.nix +++ b/nix/overlays/dataplane-dev.nix @@ -12,6 +12,10 @@ let }; in { + duvet = final.callPackage ../pkgs/duvet { + src = sources.duvet; + inherit (override-packages) rustPlatform; + }; kopium = import ../pkgs/kopium ( override-packages // { @@ -24,6 +28,9 @@ in cargo-bolero = prev.cargo-bolero.override { inherit (override-packages) rustPlatform; }; cargo-deny = prev.cargo-deny.override { inherit (override-packages) rustPlatform; }; cargo-edit = prev.cargo-edit.override { inherit (override-packages) rustPlatform; }; + cargo-expand = prev.cargo-expand.override { inherit (override-packages) rustPlatform; }; + cargo-show-asm = prev.cargo-show-asm.override { inherit (override-packages) rustPlatform; }; + cargo-mutants = prev.cargo-mutants.override { inherit (override-packages) rustPlatform; }; cargo-llvm-cov = (prev.cargo-llvm-cov.override override-packages).overrideAttrs (orig: { # the test suite is very impractical in our CI (fails on nightly for spurious reasons), and has nothing to do with # our project. diff --git a/nix/pkgs/duvet/default.nix b/nix/pkgs/duvet/default.nix new file mode 100644 index 0000000000..63d884ee65 --- /dev/null +++ b/nix/pkgs/duvet/default.nix @@ -0,0 +1,24 @@ +{ + src, + rustPlatform, + pkg-config, + openssl, + ... +}: +rustPlatform.buildRustPackage (final: { + pname = "duvet"; + version = (builtins.fromTOML (builtins.readFile "${final.src}/duvet/Cargo.toml")).package.version; + src = src.outPath; + cargoLock.lockFile = "${final.src}/Cargo.lock"; + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ openssl ]; + cargoBuildFlags = [ + "--package" + "duvet" + ]; + doCheck = false; + preBuild = '' + mkdir -p duvet/www/public + cp ${./viewer-stub.js} duvet/www/public/script.js + ''; +}) diff --git a/nix/pkgs/duvet/viewer-stub.js b/nix/pkgs/duvet/viewer-stub.js new file mode 100644 index 0000000000..96822ddc0c --- /dev/null +++ b/nix/pkgs/duvet/viewer-stub.js @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +document.getElementById("root").innerHTML = + "

This duvet was built without the bundled report viewer. The full report is embedded " + + "above as JSON, under <script id=result>.

"; diff --git a/npins/sources.json b/npins/sources.json index bd805184a7..4c6020af08 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -71,6 +71,19 @@ "url": "https://github.com/githedgehog/dplane-rpc/archive/e8fc33db10e1d00785f2a2b90cbadcad7900f200.tar.gz", "hash": "sha256-tjN4qSbKrWfosOV3wt2AnQxmVL0BPZYBjAHG3X00+aM=" }, + "duvet": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "githedgehog", + "repo": "duvet" + }, + "branch": "v0.4.3-hh", + "submodules": false, + "revision": "bcce0b50d430faf52fbfb662a28337840cd51b16", + "url": "https://github.com/githedgehog/duvet/archive/bcce0b50d430faf52fbfb662a28337840cd51b16.tar.gz", + "hash": "sha256-+f1ws/2oQ3+P3eiDHCuFyIHRn+5E/uHR+clpN0tyl3o=" + }, "fabric": { "type": "GitRelease", "repository": { diff --git a/scripts/duvet-summary.ts b/scripts/duvet-summary.ts new file mode 100755 index 0000000000..c91c0d1fed --- /dev/null +++ b/scripts/duvet-summary.ts @@ -0,0 +1,282 @@ +#!/usr/bin/env -S deno run --allow-read --allow-run --allow-write +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + + +const REPO = new URL("..", import.meta.url).pathname.replace(/\/$/, ""); + +interface Report { + specifications: Record; + annotations: { + target_path: string; + target_section: string; + level?: string; + }[]; + statuses: Record>; +} + +interface Recorded { + index: string; + spec: string; + section: string; + outcome: string; + detail?: string; + implementations: string[]; + tests: string[]; + caught: number; + missed: string[]; + unreached: string[]; + tolerated: string[]; + accepted: { mutant: string; reason: string }[]; + unviable: number; + timeout: number; +} + +interface Stats { + total: number; + complete: number; + citations: number; + tests: number; + exceptions: number; + todos: number; +} + +const EMPTY = (): Stats => ({ + total: 0, + complete: 0, + citations: 0, + tests: 0, + exceptions: 0, + todos: 0, +}); + +function record(stats: Stats, status: Record) { + const spec = status.spec; + const complete = (spec === status.citation && spec === status.test) || + spec === status.implication; + stats.total += 1; + if (!status.incomplete && (complete || spec === status.exception)) { + stats.complete += 1; + } + if (status.citation) stats.citations += 1; + if (status.test) stats.tests += 1; + if (status.exception) stats.exceptions += 1; + if (status.todo) stats.todos += 1; +} + +const BANDS = [ + { from: 0.75, cell: "๐ŸŸฉ" }, + { from: 0.5, cell: "๐ŸŸจ" }, + { from: 0.25, cell: "๐ŸŸง" }, + { from: 0, cell: "๐ŸŸฅ" }, +]; + +function bar(part: number, whole: number): string { + const ratio = whole ? part / whole : 0; + const filled = part && whole ? Math.max(1, Math.round(10 * ratio)) : 0; + const { cell } = BANDS.find((band) => ratio >= band.from) ?? BANDS[3]; + const percent = whole ? (100 * ratio).toFixed(1) : "0.0"; + return `${cell.repeat(filled)}${"โฌœ".repeat(10 - filled)} ${percent}%`; +} + +function specName(id: string): string { + const rfc = /rfc(\d+)$/.exec(id); + return rfc ? `RFC ${rfc[1]}` : id.replace(/^https?:\/\ +} + +const sectionName = (id: string) => + id.replace(/^section-/, "").replace(/^appendix-/, ""); + +const VERDICT: Record = { + held: "โœ… held", + decorative: "โŒ decorative", + uncovered: "โŒ uncovered", + stale: "โŒ stale", + failing: "โŒ failing", + error: "โš ๏ธ error", + "no-mutants": "โš ๏ธ no mutants", + unsupported: "โš ๏ธ unsupported", +}; + +async function run(cmd: string, args: string[]): Promise { + const { code } = await new Deno.Command(cmd, { + args, + cwd: REPO, + stdout: "piped", + stderr: "piped", + }).output(); + return code; +} + +function parseArgs(argv: string[]) { + const args = { json: "/tmp/duvet-summary.json", results: "", help: false }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + const value = () => { + const next = argv[++i]; + if (next === undefined) throw new Error(`${flag} needs a value`); + return next; + }; + switch (flag) { + case "--json": + args.json = value(); + break; + case "--results": + args.results = value(); + break; + case "--help": + args.help = true; + break; + default: + throw new Error(`unknown argument ${flag}`); + } + } + return args; +} + +async function main(): Promise { + const args = parseArgs(Deno.args); + if (args.help) { + console.log("usage: duvet-summary.ts [--results ] [--json ]"); + console.log( + " --results takes what `spec-interlock.ts --results` wrote; without it the", + ); + console.log(" interlock table says the run has not happened rather than"); + console.log(" reporting citations nobody has checked as though they held"); + return 0; + } + + if (await run("duvet", ["report", "--json", args.json])) { + console.error("error: duvet report failed; nothing to summarise"); + return 1; + } + const report: Report = JSON.parse(await Deno.readTextFile(args.json)); + + const lines: string[] = []; + const out = (line = "") => lines.push(line); + + const total = EMPTY(); + const rows: string[] = []; + for (const [id, spec] of Object.entries(report.specifications)) { + const stats = EMPTY(); + for (const requirement of spec.requirements) { + const status = report.statuses[String(requirement)]; + if (status) record(stats, status); + } + if (!stats.total) continue; + for (const key of Object.keys(stats) as (keyof Stats)[]) { + total[key] += stats[key]; + } + rows.push( + `| [${ + specName(id) + }](${id}) | ${stats.total} | ${stats.complete} | ${stats.citations} | ${stats.tests} | ${stats.todos} | ${stats.exceptions} | ${ + bar(stats.complete, stats.total) + } |`, + ); + } + + if (!total.total) { + console.error( + "error: duvet matched no requirement in this tree; there is nothing to summarise", + ); + return 1; + } + + out("## Specification compliance"); + out(); + out( + "| Specification | Requirements | Complete | Cited | Tested | TODO | Exceptions | Coverage |", + ); + out("| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |"); + rows.forEach(out); + out( + `| **all** | **${total.total}** | **${total.complete}** | **${total.citations}** | **${total.tests}** | **${total.todos}** | **${total.exceptions}** | ${ + bar(total.complete, total.total) + } |`, + ); + out(); + + out("## Citation interlock"); + out(); + if (!args.results) { + out( + "The interlock has not run for this summary, so no citation here is known to be more than a comment. Run `just spec-interlock --results ` and pass it with `--results`.", + ); + console.log(lines.join("\n")); + return 0; + } + + const recorded: Recorded[] = JSON.parse( + await Deno.readTextFile(args.results), + ).requirements ?? []; + const rank = ( + r: Recorded, + ) => (r.outcome === "held" ? 2 : r.outcome === "decorative" ? 0 : 1); + recorded.sort((a, b) => rank(a) - rank(b) || a.index.localeCompare(b.index)); + + const held = recorded.filter((r) => r.outcome === "held").length; + out( + `${held} of ${recorded.length} requirements carrying both citations hold them: the cited tests notice a change to the cited code.`, + ); + out(); + out("| Requirement | Implementation | Tests | Verdict |"); + out("| --- | --- | --- | --- |"); + for (const r of recorded) { + const where = `[${specName(r.spec)} ยง${ + sectionName(r.section) + }](${r.spec}#${r.section})
\`--only ${r.index}\``; + const code = r.implementations.map((i) => `\`${i}\``).join("
"); + const tests = r.tests.map((t) => `\`${t}\``).join("
"); + out( + `| ${where} | ${code} | ${tests} | ${VERDICT[r.outcome] ?? r.outcome} |`, + ); + } + out(); + + for (const r of recorded) { + const notes = [...r.missed, ...r.accepted.map((a) => a.mutant)]; + if (r.outcome === "held" && !r.accepted.length) continue; + if (!notes.length && !r.detail) continue; + out( + `
${specName(r.spec)} ยง${ + sectionName(r.section) + } (requirement ${r.index}) -- ${ + VERDICT[r.outcome] ?? r.outcome + }`, + ); + out(); + if (r.detail) { + out(`${r.detail}`); + out(); + } + const list = (title: string, items: string[]) => { + if (!items.length) return; + out(`${title}`); + out(); + for (const item of items) out(`- \`${item}\``); + out(); + }; + list( + "Survivors the tests never reach -- give them a different input:", + r.unreached, + ); + list( + "Survivors the tests run straight through -- give them a different assertion:", + r.tolerated, + ); + if (r.accepted.length) { + out("Accepted, with the reason recorded in `scripts/spec-interlock.ts`:"); + out(); + for (const a of r.accepted) out(`- \`${a.mutant}\` -- ${a.reason}`); + out(); + } + out("
"); + out(); + } + + console.log(lines.join("\n")); + return 0; +} + +if (import.meta.main) Deno.exit(await main()); diff --git a/scripts/gen-pins.sh b/scripts/gen-pins.sh index 850b73ec77..877cc57085 100755 --- a/scripts/gen-pins.sh +++ b/scripts/gen-pins.sh @@ -64,6 +64,8 @@ npins add github githedgehog frr-agent --branch master # floats with branch on p npins add github githedgehog dplane-rpc --branch master # floats with branch on pin bump npins add github githedgehog dplane-plugin --branch hh-master-10.6 # floats with branch on pin bump +npins add github githedgehog duvet --branch v0.4.3-hh # floats with branch on pin bump + npins add github opengrep opengrep npins add github mermaid-js mermaid --release-prefix "mermaid@" npins add github KaTeX KaTeX diff --git a/scripts/spec-interlock.ts b/scripts/spec-interlock.ts new file mode 100755 index 0000000000..3aeb65762e --- /dev/null +++ b/scripts/spec-interlock.ts @@ -0,0 +1,842 @@ +#!/usr/bin/env -S deno run --allow-read --allow-run --allow-write +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + + +const join = (...parts: string[]) => parts.join("/").replaceAll(/\/+/g, "/"); +const REPO = new URL("..", import.meta.url).pathname.replace(/\/$/, ""); + +const IMPLEMENTATION = "CITATION"; +const TEST = "TEST"; + +const CITATION_LINE = /^\s*\/\/[=#]/; +const ATTRIBUTE_LINE = /^\s*#\[/; +const FN_LINE = /\bfn\s+([A-Za-z_][A-Za-z0-9_]*)/; +const MUTANT_LINE = /^([^:]+):(\d+):(\d+): /; + +interface Accepted { + requirement: string; + mutant: string; + reason: string; +} + +const ACCEPTED: Accepted[] = [ + { + requirement: "https://www.rfc-editor.org/rfc/rfc4787#section-4.1", + mutant: + "nat/src/masquerade/apalloc/alloc.rs: replace match guard e.is_exhaustion() with true in IpAllocator::allocate", + reason: + "Equivalent under the allocator's invariants. `reuse_allocated_ip` skips `NoFreePort` " + + "and loops, so the only error it can return from a well-formed pool is `NoFreeIp`, " + + "which is exhaustion; the non-exhaustion arm is defensive depth against an internal " + + "inconsistency. The whole 210-test nat suite passes with the guard forced to `true`. " + + "The guard is kept because a future allocator that can fail for a non-exhaustion reason " + + "must not draw a second public address for a host that already holds one.", + }, + ...[ + "https://www.rfc-editor.org/rfc/rfc4787#section-4.2.1", + "https://www.rfc-editor.org/rfc/rfc5382#section-8", + ].map((requirement) => ({ + requirement, + mutant: + "nat/src/masquerade/apalloc/port_alloc.rs: replace < with <= in Bitmap256::allocate_port_from_bitmap", + reason: + "Equivalent on the second half, and unreachable rather than untested. `allocate_port` " + + "only enters the bitmap when `!is_full()`, and `bitmap_full()` is both halves at " + + "`u128::MAX`. Reaching the second-half branch means the first half is already full, so a " + + "block that is not full has a zero in the second half and `trailing_ones() < 128` always " + + "holds; `ones == 128`, the only value the two operators disagree on, cannot occur. The " + + "whole 210-test nat suite passes with `<=` applied. Note the same mutant on the *first* " + + "half is caught, and correctly: a block with the first half full and the second free is " + + "ordinary, so `ones == 128` is reachable there and `1u128 << 128` overflows.", + })), +]; + +function stableName(mutant: string): string { + const match = MUTANT_LINE.exec(mutant); + if (!match) return mutant; + return `${match[1]}: ${mutant.slice(match[0].length)}`; +} + +interface Region { + path: string; + start: number; + end: number; +} + +const showRegion = (r: Region) => `${r.path}:${r.start}-${r.end - 1}`; + +interface Triple { + index: string; + spec: string; + section: string; + implementations: Region[]; + tests: string[]; + testSites: Region[]; +} + +interface Annotation { + source: string; + target_path: string; + target_section: string; + line: number; + type?: string; +} + +interface Status { + citation?: number; + test?: number; + related?: number[]; +} + +interface Report { + annotations: Annotation[]; + statuses: Record; +} + +async function run( + cmd: string, + args: string[], + env: Record = {}, +): Promise<{ code: number; stdout: string; stderr: string }> { + const output = await new Deno.Command(cmd, { + args, + cwd: REPO, + env, + stdout: "piped", + stderr: "piped", + }).output(); + const decode = (b: Uint8Array) => new TextDecoder().decode(b); + return { + code: output.code, + stdout: decode(output.stdout), + stderr: decode(output.stderr), + }; +} + +async function lineCounts( + pkg: string, + tests: string[], + paths: string[], + outDir: string, + sharedTarget: string, +): Promise | null> { + const env = { + LLVM_COV: join(REPO, "devroot", "bin", "llvm-cov"), + LLVM_PROFDATA: join(REPO, "devroot", "bin", "llvm-profdata"), + CARGO_TARGET_DIR: sharedTarget, + }; + await Deno.mkdir(outDir, { recursive: true }); + + const step = async (args: string[]) => { + const { code, stderr } = await run("cargo", args, env); + if (code !== 0) { + console.log(` warning: coverage step \`${args.join(" ")}\` failed`); + const tail = stderr.trim().split("\n").slice(-3).join("\n "); + if (tail) console.log(` ${tail}`); + } + return code === 0; + }; + + const jsonPath = join(outDir, "coverage.json"); + const ok = + await step(["llvm-cov", "clean", "--workspace", "--profraw-only"]) && + await step([ + "llvm-cov", + "--no-report", + "--branch", + "nextest", + "-p", + pkg, + "-E", + testFilterset(tests), + ]) && + await step(["llvm-cov", "report", "--json", "--output-path", jsonPath]); + if (!ok) return null; + + const counts = new Map(); + let report; + try { + report = JSON.parse(await Deno.readTextFile(jsonPath)); + } catch (e) { + console.log(` warning: coverage report unreadable: ${e}`); + return null; + } + for (const file of report.data?.[0]?.files ?? []) { + const match = paths.find((p) => file.filename.endsWith(p)); + if (!match) continue; + for (const [line, _col, count, hasCount] of file.segments ?? []) { + if (!hasCount) continue; + const key = `${match}:${line}`; + counts.set(key, Math.max(counts.get(key) ?? 0, count)); + } + } + return counts; +} + +async function duvetReport(jsonPath: string): Promise { + const { code } = await run("duvet", ["report", "--json", jsonPath]); + if (code !== 0) throw new Error("duvet report failed"); + return JSON.parse(await Deno.readTextFile(jsonPath)); +} + +function skipCitationBlock(lines: string[], start: number): number { + let i = start; + while (i < lines.length) { + const line = lines[i]; + const stripped = line.trim(); + if ( + stripped === "" || CITATION_LINE.test(line) || + stripped.startsWith("//") || ATTRIBUTE_LINE.test(line) + ) { + i += 1; + continue; + } + break; + } + return i; +} + +function itemExtent(lines: string[], start: number): number { + let depth = 0; + let opened = false; + let i = start; + while (i < lines.length) { + const code = lines[i].split("//")[0]; + const opens = (code.match(/\{/g) ?? []).length; + depth += opens - (code.match(/\}/g) ?? []).length; + if (opens > 0) opened = true; + i += 1; + if (opened && depth <= 0) return i; + if (!opened && code.trimEnd().endsWith(";")) return i; + } + return i; +} + +async function readLines(path: string): Promise { + return (await Deno.readTextFile(path)).split("\n"); +} + +async function implementationRegion( + source: string, + line: number, +): Promise { + const lines = await readLines(join(REPO, source)); + const start = skipCitationBlock(lines, line - 1); + return { path: source, start: start + 1, end: itemExtent(lines, start) + 1 }; +} + +async function testName( + source: string, + line: number, +): Promise<[string, Region]> { + const lines = await readLines(join(REPO, source)); + const start = skipCitationBlock(lines, line - 1); + for (let i = start; i < Math.min(start + 5, lines.length); i += 1) { + const match = FN_LINE.exec(lines[i]); + if (match) { + return [match[1], { + path: source, + start: i + 1, + end: itemExtent(lines, i) + 1, + }]; + } + } + throw new Error( + `${source}:${line}: a type=test citation does not precede a function`, + ); +} + +async function collect(report: Report): Promise { + const triples: Triple[] = []; + for (const [index, status] of Object.entries(report.statuses)) { + if (status.citation === undefined || status.test === undefined) continue; + let triple: Triple | undefined; + for (const j of status.related ?? []) { + const annotation = report.annotations[j]; + triple ??= { + index, + spec: annotation.target_path, + section: annotation.target_section, + implementations: [], + tests: [], + testSites: [], + }; + const kind = annotation.type ?? IMPLEMENTATION; + if (kind === IMPLEMENTATION) { + triple.implementations.push( + await implementationRegion(annotation.source, annotation.line), + ); + } else if (kind === TEST) { + const [name, site] = await testName(annotation.source, annotation.line); + triple.tests.push(name); + triple.testSites.push(site); + } + } + if (triple && triple.implementations.length && triple.tests.length) { + triples.push(triple); + } + } + return triples; +} + +async function packageOf(path: string): Promise { + const manifest = join(REPO, path.split("/")[0], "Cargo.toml"); + for (const line of (await Deno.readTextFile(manifest)).split("\n")) { + if (line.startsWith("name")) { + return line.split("=")[1].trim().replaceAll('"', ""); + } + } + throw new Error(`${manifest}: no package name`); +} + +function mutantFilter(regions: Region[]): string { + const byFile = new Map>(); + for (const region of regions) { + const lines = byFile.get(region.path) ?? new Set(); + for (let n = region.start; n < region.end; n += 1) lines.add(n); + byFile.set(region.path, lines); + } + const parts = [...byFile.entries()].sort().map(([path, lines]) => { + const escaped = path.replaceAll(".", "\\."); + return `${escaped}:(${[...lines].sort((a, b) => a - b).join("|")}):`; + }); + return `^(?:${parts.join("|")})`; +} + +function selectRegions(mutants: string[], regions: Region[]): string[] { + return mutants.filter((mutant) => { + const match = MUTANT_LINE.exec(mutant); + if (!match) return false; + const [, path, line] = match; + return regions.some((r) => + r.path === path && r.start <= Number(line) && Number(line) < r.end + ); + }); +} + +function testFilterset(tests: string[]): string { + return tests.map((name) => `test(/(^|::)${name}$/)`).join(" + "); +} + +interface Suite { + testcases?: Record; +} + +async function missingTests( + pkg: string, + tests: string[], +): Promise { + const { code, stdout } = await run("cargo", [ + "nextest", + "list", + "-p", + pkg, + "-E", + testFilterset(tests), + "--message-format", + "json", + ]); + if (code !== 0) return null; + let suites: Record; + try { + suites = JSON.parse(stdout)["rust-suites"] ?? {}; + } catch { + return null; + } + const selected: string[] = []; + for (const suite of Object.values(suites)) { + for (const [name, testcase] of Object.entries(suite.testcases ?? {})) { + if (testcase["filter-match"]?.status === "matches") selected.push(name); + } + } + return tests.filter((name) => + !selected.some((full) => full === name || full.endsWith(`::${name}`)) + ); +} + +const TERMINAL = new Set([ + "CaughtMutant", + "MissedMutant", + "Timeout", + "Unviable", +]); + +const COMPLETED = new Set([0, 2, 3]); + +interface Outcome { + scenario: unknown; + summary: string; +} + +async function auditMutants( + outDir: string, + code: number, + tests: string[], +): Promise { + const readJson = async (name: string) => { + try { + return JSON.parse( + await Deno.readTextFile(join(outDir, "mutants.out", name)), + ); + } catch { + return null; + } + }; + const outcomes = await readJson("outcomes.json"); + const generated = await readJson("mutants.json"); + if (!outcomes || !Array.isArray(generated)) { + return { + outcome: "error", + detail: + `cargo mutants exited ${code} without writing a readable mutants.out; nothing was measured`, + }; + } + const all: Outcome[] = outcomes.outcomes ?? []; + const baseline = all.find((o) => o.scenario === "Baseline"); + if (!baseline) { + return { + outcome: "error", + detail: + `cargo mutants exited ${code} without recording a baseline; nothing was measured`, + }; + } + if (baseline.summary !== "Success") { + return { + outcome: "failing", + detail: `${ + tests.join(", ") + } do not pass against unmutated code (baseline ${baseline.summary}); no mutant says anything until they do`, + }; + } + const tested = all.filter((o) => o.scenario !== "Baseline"); + if (tested.length !== generated.length) { + return { + outcome: "error", + detail: + `cargo mutants exited ${code} having tested ${tested.length} of the ${generated.length} mutants it generated; the buckets are a partial run`, + }; + } + const unclassified = tested.filter((o) => !TERMINAL.has(o.summary)); + if (unclassified.length) { + return { + outcome: "error", + detail: `cargo mutants left ${unclassified.length} mutants in ${ + [...new Set(unclassified.map((o) => o.summary))].join(", ") + }, which is not a result this tool knows how to read`, + }; + } + if (!COMPLETED.has(code)) { + return { + outcome: "error", + detail: + `cargo mutants exited ${code}, which does not mean a completed run; the buckets are not evidence`, + }; + } + return null; +} + +interface Result { + outcome: + | "held" + | "decorative" + | "uncovered" + | "stale" + | "unsupported" + | "no-mutants" + | "failing" + | "error"; + detail?: string; + caught?: string[]; + missed?: string[]; + accepted?: Accepted[]; + unreached?: string[]; + tolerated?: string[]; + unviable?: string[]; + timeout?: string[]; +} + +async function runTriple( + triple: Triple, + output: string, + jobs: number, + used: Set, +): Promise { + const pkg = await packageOf(triple.implementations[0].path); + const testPackages = new Set( + await Promise.all(triple.testSites.map((s) => packageOf(s.path))), + ); + if (testPackages.size !== 1 || !testPackages.has(pkg)) { + return { + outcome: "unsupported", + detail: `implementation in ${pkg}, tests in ${ + [...testPackages].join(", ") + }`, + }; + } + const missing = await missingTests(pkg, triple.tests); + if (missing === null) { + return { + outcome: "error", + detail: + `cargo nextest list failed for ${pkg}; the citation was not checked`, + }; + } + if (missing.length) { + return { + outcome: "stale", + detail: `${pkg} has no test named ${missing.join(", ")}`, + }; + } + + const outDir = join(output, `requirement-${triple.index}`); + const files = [...new Set(triple.implementations.map((r) => r.path))].sort(); + + const counts = await lineCounts( + pkg, + triple.tests, + files, + outDir, + join(output, "cov-target"), + ); + const regionLines = triple.implementations.flatMap((r) => + Array.from( + { length: r.end - r.start }, + (_, i) => `${r.path}:${r.start + i}`, + ) + ); + const executable = regionLines.filter((key) => counts?.has(key)); + const executed = executable.filter((key) => (counts?.get(key) ?? 0) > 0); + if (counts && executable.length && !executed.length) { + return { + outcome: "uncovered", + detail: `${triple.tests.join(", ")} never executes ${ + triple.implementations.map(showRegion).join(", ") + }; the citation cannot be testing this requirement`, + }; + } + const mutation = await run("cargo", [ + "mutants", + "--package", + pkg, + "--test-tool", + "nextest", + ...files.flatMap((path) => ["--file", path]), + "--re", + mutantFilter(triple.implementations), + "--output", + outDir, + "--jobs", + String(jobs), + "--no-times", + "--", + "-E", + testFilterset(triple.tests), + ]); + const failure = await auditMutants(outDir, mutation.code, triple.tests); + if (failure) return failure; + + const read = async (name: string): Promise => { + const path = join(outDir, "mutants.out", `${name}.txt`); + let text: string; + try { + text = await Deno.readTextFile(path); + } catch { + return []; + } + return selectRegions( + text.split("\n").filter((l) => l.trim() !== ""), + triple.implementations, + ); + }; + + const [caught, survived, unviable, timeout] = await Promise.all( + ["caught", "missed", "unviable", "timeout"].map(read), + ); + + const accepted: Accepted[] = []; + const missed = survived.filter((mutant) => { + const entry = ACCEPTED.find((a) => + a.requirement === `${triple.spec}#${triple.section}` && + a.mutant === stableName(mutant) + ); + if (entry) { + accepted.push(entry); + used.add(entry); + return false; + } + return true; + }); + if (!caught.length && !missed.length && !accepted.length) { + const why = unviable.length || timeout.length + ? `${unviable.length} unviable and ${timeout.length} timed out, so none could be tested` + : "generated no mutants"; + return { + outcome: "no-mutants", + detail: `the cited region ${ + triple.implementations.map(showRegion).join(", ") + } ${why}; the citation cannot be checked this way`, + unviable, + timeout, + }; + } + const reached = (mutant: string) => { + const match = MUTANT_LINE.exec(mutant); + return match ? (counts!.get(`${match[1]}:${match[2]}`) ?? 0) > 0 : false; + }; + return { + outcome: missed.length ? "decorative" : "held", + caught, + missed, + accepted, + unreached: counts ? missed.filter((m) => !reached(m)) : [], + tolerated: counts ? missed.filter(reached) : [], + unviable, + timeout, + }; +} + +interface Recorded { + index: string; + spec: string; + section: string; + outcome: Result["outcome"]; + detail?: string; + implementations: string[]; + tests: string[]; + caught: number; + missed: string[]; + unreached: string[]; + tolerated: string[]; + accepted: { mutant: string; reason: string }[]; + unviable: number; + timeout: number; +} + +function parseArgs(argv: string[]) { + const args = { + list: false, + help: false, + only: [] as string[], + jobs: 4, + output: join(REPO, "target", "spec-interlock"), + json: "/tmp/duvet-interlock.json", + results: "", + }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + const value = () => { + const next = argv[++i]; + if (next === undefined) throw new Error(`${flag} needs a value`); + return next; + }; + switch (flag) { + case "--list": + args.list = true; + break; + case "--help": + args.help = true; + break; + case "--only": + args.only.push(value()); + break; + case "--jobs": + args.jobs = Number(value()); + break; + case "--output": + args.output = value(); + break; + case "--json": + args.json = value(); + break; + case "--results": + args.results = value(); + break; + default: + throw new Error(`unknown argument ${flag}`); + } + } + return args; +} + +async function main(): Promise { + const args = parseArgs(Deno.args); + if (args.help) { + console.log( + "usage: spec-interlock.ts [--list] [--only ]... [--jobs N]", + ); + console.log( + " [--results ] records the verdicts as JSON for scripts/duvet-summary.ts", + ); + console.log( + " results and the instrumented build are kept under target/spec-interlock; the build is", + ); + console.log( + " the expensive part of a run and every requirement in a package reuses it", + ); + return 0; + } + + const report = await duvetReport(args.json); + let triples = await collect(report); + if (args.only.length) { + triples = triples.filter((t) => args.only.includes(t.index)); + const unmatched = args.only.filter((id) => + !triples.some((t) => t.index === id) + ); + if (unmatched.length) { + console.error( + `error: no requirement carrying both citations is numbered ${ + unmatched.join(", ") + }; \`--list\` prints the numbers`, + ); + return 1; + } + } + + if (args.list) { + for (const triple of triples) { + console.log( + `${triple.spec}#${triple.section} (requirement ${triple.index})`, + ); + for (const region of triple.implementations) { + console.log(` implementation ${showRegion(region)}`); + } + triple.tests.forEach((name, i) => { + console.log( + ` test ${name} [${showRegion(triple.testSites[i])}]`, + ); + }); + console.log(); + } + console.log( + `${triples.length} requirements carry both an implementation and a test`, + ); + return 0; + } + + if (!triples.length) { + console.error( + "error: no requirement carries both an implementation and a test citation; there is nothing to check", + ); + return 1; + } + + await Deno.mkdir(args.output, { recursive: true }); + const used = new Set(); + const recorded: Recorded[] = []; + let failures = 0; + for (const triple of triples) { + console.log( + `==> ${triple.spec}#${triple.section} (requirement ${triple.index})`, + ); + const result = await runTriple( + triple, + args.output, + Number(args.jobs), + used, + ); + if (result.outcome === "held") { + console.log( + ` held: ${result.caught!.length} of ${ + result.caught!.length + result.accepted!.length + } mutants in the cited region caught by ${triple.tests.join(", ")}${ + result.accepted!.length + ? `, ${result.accepted!.length} accepted below` + : "" + }`, + ); + } else if (result.outcome === "decorative") { + failures += 1; + console.log( + ` DECORATIVE: ${result.missed!.length} mutants survive ${ + triple.tests.join(", ") + }`, + ); + if (result.unreached?.length) { + console.log( + ` unreached (${result.unreached.length}) -- the test never runs these lines, so`, + ); + console.log( + ` closing them means changing what it feeds, not what it asserts:`, + ); + for (const mutant of result.unreached) console.log(` ${mutant}`); + } + if (result.tolerated?.length) { + console.log( + ` tolerated (${result.tolerated.length}) -- the test runs these lines and passes`, + ); + console.log( + ` anyway, so either an assertion is missing or the mutant is equivalent:`, + ); + for (const mutant of result.tolerated) console.log(` ${mutant}`); + } + if (!result.unreached?.length && !result.tolerated?.length) { + console.log(` (unclassified -- coverage was not collected)`); + for (const mutant of result.missed!) console.log(` ${mutant}`); + } + } else { + failures += 1; + console.log(` ${result.outcome.toUpperCase()}: ${result.detail}`); + } + const skipped = (result.unviable?.length ?? 0) + + (result.timeout?.length ?? 0); + if (skipped && result.outcome !== "no-mutants") { + console.log( + ` (${skipped} unviable or timed out, not counted either way)`, + ); + } + recorded.push({ + index: triple.index, + spec: triple.spec, + section: triple.section, + outcome: result.outcome, + detail: result.detail, + implementations: triple.implementations.map(showRegion), + tests: triple.tests, + caught: result.caught?.length ?? 0, + missed: result.missed ?? [], + unreached: result.unreached ?? [], + tolerated: result.tolerated ?? [], + accepted: (result.accepted ?? []).map(({ mutant, reason }) => ({ + mutant, + reason, + })), + unviable: result.unviable?.length ?? 0, + timeout: result.timeout?.length ?? 0, + }); + for (const entry of result.accepted ?? []) { + console.log(` accepted: ${entry.mutant}`); + console.log(` ${entry.reason}`); + } + } + + const stale = ACCEPTED.filter((entry) => !used.has(entry)); + const checkedAll = !args.only.length; + if (stale.length && checkedAll) { + failures += stale.length; + console.log(); + for (const entry of stale) { + console.log(`STALE ACCEPT: no surviving mutant matches`); + console.log(` requirement ${entry.requirement}`); + console.log(` mutant ${entry.mutant}`); + } + } + + if (args.results) { + await Deno.writeTextFile( + args.results, + `${JSON.stringify({ requirements: recorded }, null, 2)}\n`, + ); + } + + console.log(); + console.log( + `${ + triples.length - failures + }/${triples.length} requirements hold their citations`, + ); + return failures ? 1 : 0; +} + +if (import.meta.main) Deno.exit(await main());