diff --git a/client/src/main/java-mvnd/org/mvndaemon/mvnd/client/DefaultClient.java b/client/src/main/java-mvnd/org/mvndaemon/mvnd/client/DefaultClient.java index c151c2b4f..084c5e2ca 100644 --- a/client/src/main/java-mvnd/org/mvndaemon/mvnd/client/DefaultClient.java +++ b/client/src/main/java-mvnd/org/mvndaemon/mvnd/client/DefaultClient.java @@ -152,7 +152,8 @@ public static void main(String[] argv) throws Exception { int exitCode = 0; boolean noBuffering = batchMode || parameters.noBuffering(); - try (TerminalOutput output = new TerminalOutput(noBuffering, parameters.rollingWindowSize(), logFile)) { + try (TerminalOutput output = new TerminalOutput( + noBuffering, parameters.hideBannedProjectSkips(), parameters.rollingWindowSize(), logFile)) { try { // Color // We need to defer this part until the terminal is created diff --git a/client/src/main/java/org/mvndaemon/mvnd/client/DaemonParameters.java b/client/src/main/java/org/mvndaemon/mvnd/client/DaemonParameters.java index 31a784ac9..7e23e4efa 100644 --- a/client/src/main/java/org/mvndaemon/mvnd/client/DaemonParameters.java +++ b/client/src/main/java/org/mvndaemon/mvnd/client/DaemonParameters.java @@ -383,6 +383,10 @@ public boolean noBuffering() { return property(Environment.MVND_NO_BUFERING).orFail().asBoolean(); } + public boolean hideBannedProjectSkips() { + return property(Environment.MVND_HIDE_BANNED_PROJECT_SKIPS).orFail().asBoolean(); + } + public int rollingWindowSize() { return property(Environment.MVND_ROLLING_WINDOW_SIZE).orFail().asInt(); } diff --git a/common/src/main/java/org/mvndaemon/mvnd/common/Environment.java b/common/src/main/java/org/mvndaemon/mvnd/common/Environment.java index 1737d2aef..ee8d377cc 100644 --- a/common/src/main/java/org/mvndaemon/mvnd/common/Environment.java +++ b/common/src/main/java/org/mvndaemon/mvnd/common/Environment.java @@ -161,6 +161,21 @@ public enum Environment { */ MVND_NO_MODEL_CACHE("mvnd.noModelCache", null, Boolean.FALSE, OptionType.BOOLEAN, Flags.OPTIONAL), + /** + * If true (default), mvnd shows live per-test progress on each project's worker line while + * Surefire/Failsafe run tests. Set to false to disable the feature entirely (nothing is injected + * into the surefire/failsafe configuration and no listener is registered). + */ + MVND_TEST_PROGRESS("mvnd.testProgress", null, Boolean.TRUE, OptionType.BOOLEAN, Flags.OPTIONAL), + + /** + * If true (default), the client omits the per-project + * Skipping X / This project has been banned from the build due to previous failures. blocks that + * Maven logs after a reactor failure. Set to false to show them. The final reactor summary (including + * its ... SKIPPED rows) is always kept. + */ + MVND_HIDE_BANNED_PROJECT_SKIPS("mvnd.hideBannedProjectSkips", null, Boolean.TRUE, OptionType.BOOLEAN, Flags.NONE), + /** * If true, the daemon will be launched in debug mode with the following JVM argument: * -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000; otherwise the debug argument is diff --git a/common/src/main/java/org/mvndaemon/mvnd/common/Message.java b/common/src/main/java/org/mvndaemon/mvnd/common/Message.java index 86d45bea5..8d8261241 100644 --- a/common/src/main/java/org/mvndaemon/mvnd/common/Message.java +++ b/common/src/main/java/org/mvndaemon/mvnd/common/Message.java @@ -26,6 +26,7 @@ import java.io.StringWriter; import java.io.UTFDataFormatException; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; @@ -66,9 +67,9 @@ public abstract class Message { public static final int REQUEST_INPUT = 27; public static final int INPUT_DATA = 28; /** - * Live per-test progress for a project's line while surefire/failsafe run. - * TODO: the daemon-side feed that emits this message is not implemented on mvnd-1.x yet; until it - * lands in a follow-up commit, the client render stays dormant (no test-progress suffix is shown). + * Live per-test progress for a project's line while surefire/failsafe run. Emitted by the forked test JVM's + * listener bridge, relayed through the daemon, and rendered by the client as a test-progress suffix on the + * project's status line. */ public static final int PROJECT_TEST_PROGRESS = 29; @@ -627,46 +628,87 @@ public void write(DataOutputStream output) throws IOException { public static class ProjectTestProgressEvent extends Message { final String projectId; + final int forkChannelId; final String testClass; final String testMethod; final int completed; final int failures; final int errors; final int skipped; + final int retrying; + final int flaky; + final List flakyTests; + final List failedTests; + final List erroredTests; public static ProjectTestProgressEvent read(DataInputStream input) throws IOException { final String projectId = readUTF(input); + final int forkChannelId = input.readInt(); final String testClass = readUTF(input); final String testMethod = readUTF(input); final int completed = input.readInt(); final int failures = input.readInt(); final int errors = input.readInt(); final int skipped = input.readInt(); - return new ProjectTestProgressEvent(projectId, testClass, testMethod, completed, failures, errors, skipped); + final int retrying = input.readInt(); + final int flaky = input.readInt(); + final List flakyTests = readStringList(input); + final List failedTests = readStringList(input); + final List erroredTests = readStringList(input); + return new ProjectTestProgressEvent( + projectId, + forkChannelId, + testClass, + testMethod, + completed, + failures, + errors, + skipped, + retrying, + flaky, + flakyTests, + failedTests, + erroredTests); } public ProjectTestProgressEvent( String projectId, + int forkChannelId, String testClass, String testMethod, int completed, int failures, int errors, - int skipped) { + int skipped, + int retrying, + int flaky, + List flakyTests, + List failedTests, + List erroredTests) { super(PROJECT_TEST_PROGRESS); this.projectId = Objects.requireNonNull(projectId, "projectId cannot be null"); + this.forkChannelId = forkChannelId; this.testClass = testClass; this.testMethod = testMethod; this.completed = completed; this.failures = failures; this.errors = errors; this.skipped = skipped; + this.retrying = retrying; + this.flaky = flaky; + this.flakyTests = flakyTests == null ? new ArrayList<>() : new ArrayList<>(flakyTests); + this.failedTests = failedTests == null ? new ArrayList<>() : new ArrayList<>(failedTests); + this.erroredTests = erroredTests == null ? new ArrayList<>() : new ArrayList<>(erroredTests); } public String getProjectId() { return projectId; } + public int getForkChannelId() { + return forkChannelId; + } + public String getTestClass() { return testClass; } @@ -691,26 +733,109 @@ public int getSkipped() { return skipped; } + public int getRetrying() { + return retrying; + } + + public int getFlaky() { + return flaky; + } + + public List getFlakyTests() { + return Collections.unmodifiableList(flakyTests); + } + + public List getFailedTests() { + return Collections.unmodifiableList(failedTests); + } + + public List getErroredTests() { + return Collections.unmodifiableList(erroredTests); + } + @Override public void write(DataOutputStream output) throws IOException { super.write(output); writeUTF(output, projectId); + output.writeInt(forkChannelId); writeUTF(output, testClass); writeUTF(output, testMethod); output.writeInt(completed); output.writeInt(failures); output.writeInt(errors); output.writeInt(skipped); + output.writeInt(retrying); + output.writeInt(flaky); + writeStringList(output, flakyTests); + writeStringList(output, failedTests); + writeStringList(output, erroredTests); } @Override public String toString() { - return "ProjectTestProgress{projectId='" + projectId + "', testClass='" + testClass + "', testMethod='" - + testMethod + "', completed=" + completed + ", failures=" + failures + ", errors=" + errors - + ", skipped=" + skipped + "}"; + return "ProjectTestProgress{projectId='" + projectId + "', forkChannelId=" + forkChannelId + + ", testClass='" + testClass + "', testMethod='" + testMethod + "', completed=" + completed + + ", failures=" + failures + ", errors=" + errors + ", skipped=" + skipped + + ", retrying=" + retrying + ", flaky=" + flaky + ", flakyTests=" + flakyTests + + ", failedTests=" + failedTests + ", erroredTests=" + erroredTests + "}"; } } + public static ProjectTestProgressEvent projectTestProgress( + String projectId, + int forkChannelId, + String testClass, + String testMethod, + int completed, + int failures, + int errors, + int skipped) { + return projectTestProgress( + projectId, + forkChannelId, + testClass, + testMethod, + completed, + failures, + errors, + skipped, + 0, + 0, + null, + null, + null); + } + + public static ProjectTestProgressEvent projectTestProgress( + String projectId, + int forkChannelId, + String testClass, + String testMethod, + int completed, + int failures, + int errors, + int skipped, + int retrying, + int flaky, + List flakyTests, + List failedTests, + List erroredTests) { + return new ProjectTestProgressEvent( + projectId, + forkChannelId, + testClass, + testMethod, + completed, + failures, + errors, + skipped, + retrying, + flaky, + flakyTests, + failedTests, + erroredTests); + } + public static ProjectTestProgressEvent projectTestProgress( String projectId, String testClass, @@ -719,7 +844,36 @@ public static ProjectTestProgressEvent projectTestProgress( int failures, int errors, int skipped) { - return new ProjectTestProgressEvent(projectId, testClass, testMethod, completed, failures, errors, skipped); + return projectTestProgress(projectId, -1, testClass, testMethod, completed, failures, errors, skipped); + } + + public static ProjectTestProgressEvent projectTestProgress( + String projectId, + String testClass, + String testMethod, + int completed, + int failures, + int errors, + int skipped, + int retrying, + int flaky, + List flakyTests, + List failedTests, + List erroredTests) { + return projectTestProgress( + projectId, + -1, + testClass, + testMethod, + completed, + failures, + errors, + skipped, + retrying, + flaky, + flakyTests, + failedTests, + erroredTests); } public static class BuildStarted extends Message { diff --git a/common/src/main/java/org/mvndaemon/mvnd/common/logging/TerminalOutput.java b/common/src/main/java/org/mvndaemon/mvnd/common/logging/TerminalOutput.java index cc9f1818e..e7397ab42 100644 --- a/common/src/main/java/org/mvndaemon/mvnd/common/logging/TerminalOutput.java +++ b/common/src/main/java/org/mvndaemon/mvnd/common/logging/TerminalOutput.java @@ -30,12 +30,15 @@ import java.util.Collections; import java.util.Deque; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; +import java.util.regex.Pattern; import java.util.stream.Collector; import java.util.stream.Collectors; @@ -141,6 +144,11 @@ public class TerminalOutput implements ClientOutput { private String buildStatus; private boolean displayDone = false; private boolean noBuffering; + private final Map failureProgress = new LinkedHashMap<>(); + /** When {@code true}, "Skipping X / banned from the build" reactor blocks are dropped from the console. */ + private final boolean hideBannedProjectSkips; + + private final BannedSkipFilter bannedSkipFilter = new BannedSkipFilter(); /** * {@link Project} is owned by the display loop thread and is accessed only from there. Therefore it does not need @@ -149,7 +157,7 @@ public class TerminalOutput implements ClientOutput { static class Project { final String id; MojoStartedEvent runningExecution; - Message.ProjectTestProgressEvent testProgress; + final Map testProgress = new LinkedHashMap<>(); final List log = new ArrayList<>(); public Project(String id) { @@ -158,12 +166,18 @@ public Project(String id) { } public TerminalOutput(boolean noBuffering, int rollingWindowSize, Path logFile) throws IOException { + this(noBuffering, true, rollingWindowSize, logFile); + } + + public TerminalOutput(boolean noBuffering, boolean hideBannedProjectSkips, int rollingWindowSize, Path logFile) + throws IOException { this.start = System.currentTimeMillis(); TerminalBuilder builder = TerminalBuilder.builder(); builder.systemOutput(TerminalBuilder.SystemOutput.SysErr); this.terminal = builder.build(); this.dumb = terminal.getType().startsWith("dumb"); this.noBuffering = noBuffering; + this.hideBannedProjectSkips = hideBannedProjectSkips; this.linesPerProject = rollingWindowSize; terminal.enterRawMode(); Thread mainThread = Thread.currentThread(); @@ -240,6 +254,9 @@ private boolean doAccept(Message entry) { break; } case Message.CANCEL_BUILD: { + if (hideBannedProjectSkips) { + bannedSkipFilter.flush(log::accept); + } projects.values().stream().flatMap(p -> p.log.stream()).forEach(log); clearDisplay(); try { @@ -260,6 +277,9 @@ private boolean doAccept(Message entry) { } else { msg = e.getClassName() + ": " + e.getMessage(); } + if (hideBannedProjectSkips) { + bannedSkipFilter.flush(log::accept); + } projects.values().stream().flatMap(p -> p.log.stream()).forEach(log); clearDisplay(); try { @@ -282,7 +302,7 @@ private boolean doAccept(Message entry) { final MojoStartedEvent execution = (MojoStartedEvent) entry; final Project prj = projects.computeIfAbsent(execution.getArtifactId(), Project::new); prj.runningExecution = execution; - prj.testProgress = null; + prj.testProgress.clear(); break; } case Message.PROJECT_STOPPED: { @@ -301,6 +321,9 @@ private boolean doAccept(Message entry) { break; } case Message.BUILD_FINISHED: { + if (hideBannedProjectSkips) { + bannedSkipFilter.flush(log::accept); + } projects.values().stream().flatMap(p -> p.log.stream()).forEach(log); clearDisplay(); try { @@ -387,14 +410,14 @@ private boolean doAccept(Message entry) { } case Message.BUILD_LOG_MESSAGE: { StringMessage sm = (StringMessage) entry; - log.accept(sm.getMessage()); + acceptReactorLine(sm.getMessage()); break; } case Message.PROJECT_LOG_MESSAGE: { final ProjectEvent bm = (ProjectEvent) entry; final Project prj = projects.get(bm.getProjectId()); if (prj == null) { - log.accept(bm.getMessage()); + acceptReactorLine(bm.getMessage()); } else if (noBuffering || dumb) { String msg; if (maxThreads > 1) { @@ -456,6 +479,13 @@ private boolean doAccept(Message entry) { case Message.EXECUTION_FAILURE: { final ExecutionFailureEvent efe = (ExecutionFailureEvent) entry; failures.add(efe); + final Project prj = projects.get(efe.getProjectId()); + if (prj != null) { + Message.ProjectTestProgressEvent tp = aggregateTestProgress(prj.testProgress.values()); + if (tp != null) { + failureProgress.put(efe.getProjectId(), tp); + } + } break; } case Message.REQUEST_INPUT: { @@ -471,7 +501,7 @@ private boolean doAccept(Message entry) { final Message.ProjectTestProgressEvent e = (Message.ProjectTestProgressEvent) entry; final Project prj = projects.get(e.getProjectId()); if (prj != null) { - prj.testProgress = e; + prj.testProgress.put(e.getForkChannelId(), e); } break; } @@ -625,39 +655,43 @@ private void update() { dispLines--; } - if (projectsCount <= dispLines) { - int remLogLines = dispLines - projectsCount; - for (Project prj : projects.values()) { - addProjectLine(lines, prj); - // get the last lines of the project log, taking multi-line logs into account - int nb = Math.min(remLogLines, linesPerProject); - List logs = lastN(prj.log, nb).stream() - .flatMap(s -> AttributedString.fromAnsi(s).columnSplitLength(Integer.MAX_VALUE).stream()) - .map(s -> concat(" ", s)) - .collect(lastN(nb)); - lines.addAll(logs); - remLogLines -= logs.size(); - } - final AttributedString idleLine = new AttributedStringBuilder() - .style(BOLD_GREEN_FOREGROUND) - .append("> ") - .style(AttributedStyle.DEFAULT.faint()) - .append("IDLE") - .style(AttributedStyle.DEFAULT) - .toAttributedString(); - int idleSlots = maxThreads - projectsCount; - while (idleSlots-- > 0 && remLogLines-- > 0 && lines.size() <= maxThreads + 1) { - lines.add(idleLine); - } - } else { - int skipProjects = projectsCount - dispLines; - for (Project prj : projects.values()) { - if (skipProjects == 0) { + if (shouldShowProjectDetails(projectsCount, dispLines, failures.size())) { + if (projectsCount <= dispLines) { + int remLogLines = dispLines - projectsCount; + for (Project prj : projects.values()) { addProjectLine(lines, prj); - } else { - skipProjects--; + // get the last lines of the project log, taking multi-line logs into account + int nb = Math.min(remLogLines, linesPerProject); + List logs = lastN(prj.log, nb).stream() + .flatMap(s -> AttributedString.fromAnsi(s).columnSplitLength(Integer.MAX_VALUE).stream()) + .map(s -> concat(" ", s)) + .collect(lastN(nb)); + lines.addAll(logs); + remLogLines -= logs.size(); + } + final AttributedString idleLine = new AttributedStringBuilder() + .style(BOLD_GREEN_FOREGROUND) + .append("> ") + .style(AttributedStyle.DEFAULT.faint()) + .append("IDLE") + .style(AttributedStyle.DEFAULT) + .toAttributedString(); + int idleSlots = maxThreads - projectsCount; + while (idleSlots-- > 0 && remLogLines-- > 0 && lines.size() <= maxThreads + 1) { + lines.add(idleLine); + } + } else { + int skipProjects = projectsCount - dispLines; + for (Project prj : projects.values()) { + if (skipProjects == 0) { + addProjectLine(lines, prj); + } else { + skipProjects--; + } } } + } else { + // On large failing reactors, keep the summary visible and stop churning project lines. } List trimmed = lines.stream().map(s -> s.columnSubSequence(0, cols)).collect(Collectors.toList()); @@ -686,6 +720,10 @@ private AttributedString formatFailures() { } asb.append(": ").append(exception); } + Message.ProjectTestProgressEvent tp = failureProgress.get(efe.getProjectId()); + if (tp != null) { + appendTestProgress(asb, tp); + } } else { asb.append(String.valueOf(failures.size())).append(" projects failed: "); asb.append( @@ -775,6 +813,8 @@ public static String pathToMaven(String location) { static String renderBar(int percent) { final int width = 20; + // percent is expected in [0, 100]; clamp defensively so a rounding/caller quirk can't under/overfill the bar. + percent = Math.max(0, Math.min(100, percent)); int filled = (int) Math.round(percent / 100.0 * width); StringBuilder sb = new StringBuilder(width + 2); sb.append('['); @@ -888,7 +928,7 @@ private void addProjectLine(final List lines, Project prj) { .append('(') .append(execution.getExecutionId()) .append(')'); - final Message.ProjectTestProgressEvent tp = prj.testProgress; + final Message.ProjectTestProgressEvent tp = aggregateTestProgress(prj.testProgress.values()); if (tp != null) { appendTestProgress(asb, tp); } @@ -896,9 +936,151 @@ private void addProjectLine(final List lines, Project prj) { lines.add(asb.toAttributedString()); } + /** Matches SGR (color) escape sequences emitted by the daemon-side log renderer. */ + private static final Pattern ANSI = Pattern.compile("\\[[0-9;]*m"); + /** Matches a leading {@code [LEVEL] } prefix such as {@code [INFO] } or {@code [ERROR] }. */ + private static final Pattern LEVEL_PREFIX = Pattern.compile("^\\[[A-Z]+\\]\\s?"); + + // Matched verbatim against Maven's reactor log text (no programmatic API for this exists); if Maven ever changes + // this message, hideBannedProjectSkips silently stops filtering instead of failing loud. + private static final String BANNED_MARKER = "This project has been banned from the build due to previous failures."; + + /** Strips ANSI color and the {@code [LEVEL] } prefix so reactor lines can be matched by their bare text. */ + static String stripDecoration(String line) { + if (line == null) { + return ""; + } + String s = ANSI.matcher(line).replaceAll(""); + s = LEVEL_PREFIX.matcher(s).replaceFirst(""); + return s.trim(); + } + + private static boolean isSeparator(String stripped) { + if (stripped.isEmpty()) { + return false; + } + for (int i = 0; i < stripped.length(); i++) { + if (stripped.charAt(i) != '-') { + return false; + } + } + return true; + } + + /** + * Handles a reactor-level (project-less) Maven log line: drops "banned from the build" skip blocks when enabled. + */ + private void acceptReactorLine(String line) { + acceptReactorLine(line, hideBannedProjectSkips, bannedSkipFilter, log); + } + + /** + * Processes one reactor line: drops "banned from the build" blocks when {@code hideBannedProjectSkips} is set. + * Static and side-effect free apart from {@code out}/{@code filter} so it can be unit-tested without a terminal. + */ + static void acceptReactorLine( + String line, boolean hideBannedProjectSkips, BannedSkipFilter filter, Consumer out) { + if (hideBannedProjectSkips) { + filter.accept(line, stripDecoration(line), out); + } else { + out.accept(line); + } + } + + /** + * Drops the five-line reactor block Maven logs for a banned project (blank, separator, {@code Skipping X}, + * {@code This project has been banned...}, separator) while leaving every other line, including the final + * reactor-summary {@code ... SKIPPED} rows, untouched. Blank/separator/{@code Skipping} lines are buffered so the + * preamble can be discarded retroactively once the banned marker confirms the block; buffered lines are flushed + * ahead of any real content line (order preserved) and by {@link #flush(Consumer)} at build end. + */ + static final class BannedSkipFilter { + private final List pending = new ArrayList<>(); + private boolean swallowNextSeparator; + + void accept(String line, String stripped, Consumer out) { + if (stripped.equals(BANNED_MARKER)) { + pending.clear(); // drop the buffered blank + separator + "Skipping X" preamble and this marker + swallowNextSeparator = true; + return; + } + if (swallowNextSeparator) { + swallowNextSeparator = false; + if (isSeparator(stripped)) { + return; // drop the block's closing separator + } + } + if (stripped.isEmpty() || isSeparator(stripped) || stripped.startsWith("Skipping ")) { + pending.add(line); // structural or candidate line: hold until the next real line resolves it + return; + } + flush(out); + out.accept(line); + } + + void flush(Consumer out) { + for (String held : pending) { + out.accept(held); + } + pending.clear(); + } + } + + static Message.ProjectTestProgressEvent aggregateTestProgress( + Collection snapshots) { + if (snapshots.isEmpty()) { + return null; + } + int completed = 0; + int failures = 0; + int errors = 0; + int skipped = 0; + int retrying = 0; + int flaky = 0; + Set flakyTests = new LinkedHashSet<>(); + Set failedTests = new LinkedHashSet<>(); + Set erroredTests = new LinkedHashSet<>(); + Message.ProjectTestProgressEvent latest = null; + long latestSeq = Long.MIN_VALUE; + for (Message.ProjectTestProgressEvent tp : snapshots) { + completed += tp.getCompleted(); + failures += tp.getFailures(); + errors += tp.getErrors(); + skipped += tp.getSkipped(); + retrying += tp.getRetrying(); + flaky += tp.getFlaky(); + flakyTests.addAll(tp.getFlakyTests()); + failedTests.addAll(tp.getFailedTests()); + erroredTests.addAll(tp.getErroredTests()); + if (tp.seq() > latestSeq) { + latestSeq = tp.seq(); + latest = tp; + } + } + return Message.projectTestProgress( + latest.getProjectId(), + latest.getForkChannelId(), + latest.getTestClass(), + latest.getTestMethod(), + completed, + failures, + errors, + skipped, + retrying, + flaky, + new ArrayList<>(flakyTests), + new ArrayList<>(failedTests), + new ArrayList<>(erroredTests)); + } + + static boolean shouldShowProjectDetails(int projectsCount, int dispLines, int failuresCount) { + return failuresCount == 0 || projectsCount <= dispLines; + } + static void appendTestProgress(AttributedStringBuilder asb, Message.ProjectTestProgressEvent tp) { final AttributedStyle faint = AttributedStyle.DEFAULT.faint(); final AttributedStyle red = AttributedStyle.DEFAULT.foreground(AttributedStyle.RED); + final AttributedStyle yellow = AttributedStyle.DEFAULT.foreground(AttributedStyle.YELLOW); asb.append(' ').style(faint).append("[Tests: ").append(String.valueOf(tp.getCompleted())); if (tp.getFailures() > 0) { asb.style(faint).append(", Failures: ").style(red).append(String.valueOf(tp.getFailures())); @@ -909,6 +1091,12 @@ static void appendTestProgress(AttributedStringBuilder asb, Message.ProjectTestP if (tp.getSkipped() > 0) { asb.style(faint).append(", Skipped: ").append(String.valueOf(tp.getSkipped())); } + if (tp.getRetrying() > 0) { + asb.style(faint).append(", Retrying: ").append(String.valueOf(tp.getRetrying())); + } + if (tp.getFlaky() > 0) { + asb.style(faint).append(", Flaky: ").style(yellow).append(String.valueOf(tp.getFlaky())); + } asb.style(faint).append("]"); final String testClass = tp.getTestClass(); if (testClass != null) { diff --git a/common/src/main/java/org/mvndaemon/mvnd/testprogress/MvndTestProgress.java b/common/src/main/java/org/mvndaemon/mvnd/testprogress/MvndTestProgress.java new file mode 100644 index 000000000..fbbf161f0 --- /dev/null +++ b/common/src/main/java/org/mvndaemon/mvnd/testprogress/MvndTestProgress.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.testprogress; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Bridge between Surefire's plugin realm (where {@code MvndForkNodeFactory} runs) and mvnd's daemon realm + * (where the {@code ClientDispatcher} lives). This type MUST be loaded from a package exported by the Maven core + * realm so both realms resolve the same {@link Class} and therefore share the static {@link #LISTENER} registry. + */ +public interface MvndTestProgress { + + /** + * Push a per-test progress snapshot. Implementations must be cheap and non-throwing; the caller already + * guards against exceptions but should not rely on it. + */ + void update( + String projectId, + int forkChannelId, + String testClass, + String testMethod, + int completed, + int failures, + int errors, + int skipped, + int retrying, + int flaky, + List flakyTests, + List failedTests, + List erroredTests); + + AtomicReference LISTENER = new AtomicReference<>(); + + /** Registered by the daemon at build start; cleared at build end. */ + static void setListener(MvndTestProgress listener) { + LISTENER.set(listener); + } + + /** Returns the active listener, or {@code null} when the feature is off or this is not a daemon invocation. */ + static MvndTestProgress getListener() { + return LISTENER.get(); + } +} diff --git a/common/src/test/java/org/mvndaemon/mvnd/common/MessageTest.java b/common/src/test/java/org/mvndaemon/mvnd/common/MessageTest.java index e876875c6..5197692da 100644 --- a/common/src/test/java/org/mvndaemon/mvnd/common/MessageTest.java +++ b/common/src/test/java/org/mvndaemon/mvnd/common/MessageTest.java @@ -76,7 +76,20 @@ void buildExceptionSerialization() throws Exception { @Test void projectTestProgressSerialization() throws IOException { - Message msg = Message.projectTestProgress("my-app", "com.acme.FooTest", "shouldWork", 3, 1, 0, 1); + Message msg = Message.projectTestProgress( + "my-app", + 7, + "com.acme.FooTest", + "shouldWork", + 3, + 1, + 0, + 1, + 2, + 1, + java.util.List.of("FooTest#shouldWork"), + java.util.List.of("FooTest#broken: expected <5> but was <4>"), + java.util.List.of("FooTest#blows: / by zero")); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (DataOutputStream daos = new DataOutputStream(baos)) { @@ -90,17 +103,23 @@ void projectTestProgressSerialization() throws IOException { assertTrue(msg2 instanceof Message.ProjectTestProgressEvent); Message.ProjectTestProgressEvent e = (Message.ProjectTestProgressEvent) msg2; assertEquals("my-app", e.getProjectId()); + assertEquals(7, e.getForkChannelId()); assertEquals("com.acme.FooTest", e.getTestClass()); assertEquals("shouldWork", e.getTestMethod()); assertEquals(3, e.getCompleted()); assertEquals(1, e.getFailures()); assertEquals(0, e.getErrors()); assertEquals(1, e.getSkipped()); + assertEquals(2, e.getRetrying()); + assertEquals(1, e.getFlaky()); + assertEquals(java.util.List.of("FooTest#shouldWork"), e.getFlakyTests()); + assertEquals(java.util.List.of("FooTest#broken: expected <5> but was <4>"), e.getFailedTests()); + assertEquals(java.util.List.of("FooTest#blows: / by zero"), e.getErroredTests()); } @Test void projectTestProgressNullClassAndMethod() throws IOException { - Message msg = Message.projectTestProgress("my-app", null, null, 0, 0, 0, 0); + Message msg = Message.projectTestProgress("my-app", 11, null, null, 0, 0, 0, 0); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (DataOutputStream daos = new DataOutputStream(baos)) { msg.write(daos); @@ -110,7 +129,11 @@ void projectTestProgressNullClassAndMethod() throws IOException { msg2 = Message.read(dis); } Message.ProjectTestProgressEvent e = (Message.ProjectTestProgressEvent) msg2; + assertEquals(11, e.getForkChannelId()); assertNull(e.getTestClass()); assertNull(e.getTestMethod()); + assertEquals(0, e.getRetrying()); + assertEquals(0, e.getFlaky()); + assertTrue(e.getFlakyTests().isEmpty()); } } diff --git a/common/src/test/java/org/mvndaemon/mvnd/common/logging/TerminalOutputTest.java b/common/src/test/java/org/mvndaemon/mvnd/common/logging/TerminalOutputTest.java index e517b16e9..2be185bd2 100644 --- a/common/src/test/java/org/mvndaemon/mvnd/common/logging/TerminalOutputTest.java +++ b/common/src/test/java/org/mvndaemon/mvnd/common/logging/TerminalOutputTest.java @@ -44,11 +44,18 @@ void renderBarFull() { assertEquals("[====================]", TerminalOutput.renderBar(100)); } + @Test + void renderBarClampsOutOfRangeInput() { + // doneProjects*100/totalProjects can't actually exceed [0,100], but renderBar clamps defensively anyway + assertEquals("[ ]", TerminalOutput.renderBar(-10)); + assertEquals("[====================]", TerminalOutput.renderBar(150)); + } + @Test void suffixAllPassing() { AttributedStringBuilder asb = new AttributedStringBuilder(); TerminalOutput.appendTestProgress( - asb, Message.projectTestProgress("app", "com.acme.FooTest", "shouldWork", 12, 0, 0, 0)); + asb, Message.projectTestProgress("app", 1, "com.acme.FooTest", "shouldWork", 12, 0, 0, 0)); assertEquals(" [Tests: 12] FooTest#shouldWork", asb.toAttributedString().toString()); } @@ -56,7 +63,7 @@ void suffixAllPassing() { void suffixFailuresRenderRed() { AttributedStringBuilder asb = new AttributedStringBuilder(); TerminalOutput.appendTestProgress( - asb, Message.projectTestProgress("app", "com.acme.FooTest", "shouldWork", 12, 1, 0, 0)); + asb, Message.projectTestProgress("app", 1, "com.acme.FooTest", "shouldWork", 12, 1, 0, 0)); AttributedString s = asb.toAttributedString(); assertEquals(" [Tests: 12, Failures: 1] FooTest#shouldWork", s.toString()); int failureDigit = s.toString().indexOf("Failures: ") + "Failures: ".length(); @@ -69,17 +76,146 @@ void suffixFailuresRenderRed() { void suffixErrorsAndSkips() { AttributedStringBuilder asb = new AttributedStringBuilder(); TerminalOutput.appendTestProgress( - asb, Message.projectTestProgress("app", "com.acme.FooTest", "shouldWork", 5, 0, 2, 1)); + asb, Message.projectTestProgress("app", 1, "com.acme.FooTest", "shouldWork", 5, 0, 2, 1)); assertEquals( " [Tests: 5, Errors: 2, Skipped: 1] FooTest#shouldWork", asb.toAttributedString().toString()); } + @Test + void suffixRetryingAndFlakyTests() { + AttributedStringBuilder asb = new AttributedStringBuilder(); + TerminalOutput.appendTestProgress( + asb, + Message.projectTestProgress( + "app", + 1, + "com.acme.FooTest", + "shouldWork", + 4, + 0, + 0, + 0, + 1, + 2, + java.util.List.of("FooTest#shouldWork", "FooTest#other"), + java.util.List.of(), + java.util.List.of())); + AttributedString s = asb.toAttributedString(); + assertEquals(" [Tests: 4, Retrying: 1, Flaky: 2] FooTest#shouldWork", s.toString()); + int flakyDigit = s.toString().indexOf("Flaky: ") + "Flaky: ".length(); + assertEquals(AttributedStyle.DEFAULT.foreground(AttributedStyle.YELLOW), s.styleAt(flakyDigit)); + } + @Test void suffixClassOnly() { AttributedStringBuilder asb = new AttributedStringBuilder(); TerminalOutput.appendTestProgress( - asb, Message.projectTestProgress("app", "com.acme.FooTest", null, 3, 0, 0, 0)); + asb, Message.projectTestProgress("app", 1, "com.acme.FooTest", null, 3, 0, 0, 0)); assertEquals(" [Tests: 3] FooTest", asb.toAttributedString().toString()); } + + @Test + void aggregateTestProgressSumsForkSnapshots() { + Message.ProjectTestProgressEvent failed = + Message.projectTestProgress("app", 1, "com.acme.FooTest", "failedTest", 3, 1, 0, 0); + Message.ProjectTestProgressEvent skipped = + Message.projectTestProgress("app", 2, "com.acme.FooTest", "skippedTest", 2, 0, 0, 1); + + Message.ProjectTestProgressEvent aggregated = + TerminalOutput.aggregateTestProgress(java.util.List.of(failed, skipped)); + + assertEquals(5, aggregated.getCompleted()); + assertEquals(1, aggregated.getFailures()); + assertEquals(0, aggregated.getErrors()); + assertEquals(1, aggregated.getSkipped()); + assertEquals("com.acme.FooTest", aggregated.getTestClass()); + assertEquals("skippedTest", aggregated.getTestMethod()); + } + + @Test + void hidesProjectDetailsForLargeFailingReactors() { + assertEquals(false, TerminalOutput.shouldShowProjectDetails(20, 10, 1)); + assertEquals(true, TerminalOutput.shouldShowProjectDetails(20, 10, 0)); + assertEquals(true, TerminalOutput.shouldShowProjectDetails(5, 10, 1)); + } + + @Test + void stripDecorationRemovesLevelPrefixAndAnsi() { + assertEquals("BUILD FAILURE", TerminalOutput.stripDecoration("[INFO] BUILD FAILURE")); + assertEquals("BUILD FAILURE", TerminalOutput.stripDecoration("[INFO] BUILD FAILURE")); + assertEquals( + "This project has been banned from the build due to previous failures.", + TerminalOutput.stripDecoration( + "[INFO] This project has been banned from the build due to previous failures.")); + } + + @Test + void bannedSkipFilterDropsBannedBlockButKeepsOtherLines() { + TerminalOutput.BannedSkipFilter filter = new TerminalOutput.BannedSkipFilter(); + java.util.List out = new java.util.ArrayList<>(); + String sep = "[INFO] ------------------------------------------------------------------------"; + String[] lines = { + "[INFO] Reactor Summary:", + "[INFO] ", + sep, + "[INFO] Skipping Camel :: YAML DSL", + "[INFO] This project has been banned from the build due to previous failures.", + sep, + "[INFO] camel-core ......... SKIPPED", + }; + for (String l : lines) { + filter.accept(l, TerminalOutput.stripDecoration(l), out::add); + } + filter.flush(out::add); + + assertEquals(java.util.List.of("[INFO] Reactor Summary:", "[INFO] camel-core ......... SKIPPED"), out); + } + + @Test + void acceptReactorLineDropsBannedBlockWhenSuppressionEnabled() { + String sep = "[INFO] ------------------------------------------------------------------------"; + String[] lines = { + sep, + "[INFO] Skipping Camel :: YAML DSL", + "[INFO] This project has been banned from the build due to previous failures.", + sep, + "[INFO] Reactor Summary:", + "[INFO] camel-core ......... SKIPPED", + }; + + java.util.List out = new java.util.ArrayList<>(); + TerminalOutput.BannedSkipFilter filter = new TerminalOutput.BannedSkipFilter(); + for (String l : lines) { + TerminalOutput.acceptReactorLine(l, true, filter, out::add); + } + filter.flush(out::add); + + assertEquals(java.util.List.of("[INFO] Reactor Summary:", "[INFO] camel-core ......... SKIPPED"), out); + } + + @Test + void reactorLineIsPassedThroughUnchangedWhenSuppressionDisabled() { + java.util.List out = new java.util.ArrayList<>(); + TerminalOutput.BannedSkipFilter filter = new TerminalOutput.BannedSkipFilter(); + TerminalOutput.acceptReactorLine( + "[INFO] This project has been banned from the build due to previous failures.", + false, + filter, + out::add); + assertEquals( + java.util.List.of("[INFO] This project has been banned from the build due to previous failures."), out); + } + + @Test + void bannedSkipFilterKeepsUnbannedSkippingLine() { + TerminalOutput.BannedSkipFilter filter = new TerminalOutput.BannedSkipFilter(); + java.util.List out = new java.util.ArrayList<>(); + filter.accept( + "[INFO] Skipping bad plugin", TerminalOutput.stripDecoration("[INFO] Skipping bad plugin"), out::add); + filter.accept("[INFO] Building foo", TerminalOutput.stripDecoration("[INFO] Building foo"), out::add); + filter.flush(out::add); + + assertEquals(java.util.List.of("[INFO] Skipping bad plugin", "[INFO] Building foo"), out); + } } diff --git a/daemon/pom.xml b/daemon/pom.xml index d8f0a8ee4..dbf01e692 100644 --- a/daemon/pom.xml +++ b/daemon/pom.xml @@ -46,6 +46,10 @@ + + org.apache.maven.daemon + mvnd-surefire-progress + org.apache.maven.daemon mvnd-native @@ -58,6 +62,14 @@ org.apache.maven maven-core + + + org.codehaus.plexus + plexus-xml + 4.0.4 + provided + org.apache.maven maven-embedder diff --git a/daemon/src/main/java/org/apache/maven/cli/DaemonMavenCli.java b/daemon/src/main/java/org/apache/maven/cli/DaemonMavenCli.java index ba5684792..fa4fbb215 100644 --- a/daemon/src/main/java/org/apache/maven/cli/DaemonMavenCli.java +++ b/daemon/src/main/java/org/apache/maven/cli/DaemonMavenCli.java @@ -682,6 +682,7 @@ DefaultPlexusContainer doCreateContainer(CliRequest cliRequest) throws Exception } exportedPackages.add("org.codehaus.plexus.components.interactivity"); exportedPackages.add("org.mvndaemon.mvnd.interactivity"); + exportedPackages.add("org.mvndaemon.mvnd.testprogress"); exportedArtifacts.add("org.codehaus.plexus:plexus-interactivity-api"); final CoreExports exports = new CoreExports(containerRealm, exportedArtifacts, exportedPackages); diff --git a/daemon/src/main/java/org/mvndaemon/mvnd/cache/invalidating/InvalidatingPluginRealmCache.java b/daemon/src/main/java/org/mvndaemon/mvnd/cache/invalidating/InvalidatingPluginRealmCache.java index 805866e51..f88ac2767 100644 --- a/daemon/src/main/java/org/mvndaemon/mvnd/cache/invalidating/InvalidatingPluginRealmCache.java +++ b/daemon/src/main/java/org/mvndaemon/mvnd/cache/invalidating/InvalidatingPluginRealmCache.java @@ -22,6 +22,7 @@ import javax.inject.Named; import javax.inject.Singleton; +import java.net.URL; import java.nio.file.Path; import java.util.List; import java.util.stream.Stream; @@ -36,6 +37,7 @@ import org.eclipse.sisu.Priority; import org.mvndaemon.mvnd.cache.Cache; import org.mvndaemon.mvnd.cache.CacheFactory; +import org.mvndaemon.mvnd.forknode.MvndSurefireProgressLocator; @Singleton @Named @@ -86,7 +88,9 @@ public CacheRecord get(Key key, PluginRealmSupplier supplier) try { Record r = cache.computeIfAbsent(key, k -> { try { - return new Record(supplier.load()); + CacheRecord loaded = supplier.load(); + addTestProgressJarIfSurefire(loaded.getRealm()); + return new Record(loaded); } catch (PluginResolutionException | PluginContainerException e) { throw new RuntimeException(e); } @@ -103,6 +107,28 @@ public CacheRecord get(Key key, PluginRealmSupplier supplier) } } + /** + * Puts the {@code mvnd-surefire-progress} jar on the surefire/failsafe plugin realm so Surefire can load + * {@code MvndForkNodeFactory} when it parses the injected {@code } configuration. No-op for any other + * plugin realm, and never fails plugin-realm creation because of the progress feature. + */ + private static void addTestProgressJarIfSurefire(ClassRealm realm) { + String id = realm != null ? realm.getId() : null; + if (id == null || (!id.contains("maven-surefire-plugin") && !id.contains("maven-failsafe-plugin"))) { + return; + } + try { + java.security.CodeSource cs = + MvndSurefireProgressLocator.class.getProtectionDomain().getCodeSource(); + URL jar = cs != null ? cs.getLocation() : null; + if (jar != null) { + realm.addURL(jar); + } + } catch (RuntimeException e) { + // ignore: the test-progress feature must never break plugin realm creation + } + } + @Override public CacheRecord put(Key key, ClassRealm pluginRealm, List pluginArtifacts) { CacheRecord record = super.put(key, pluginRealm, pluginArtifacts); diff --git a/daemon/src/main/java/org/mvndaemon/mvnd/daemon/ClientDispatcher.java b/daemon/src/main/java/org/mvndaemon/mvnd/daemon/ClientDispatcher.java index f9bb805c1..3191a8d9e 100644 --- a/daemon/src/main/java/org/mvndaemon/mvnd/daemon/ClientDispatcher.java +++ b/daemon/src/main/java/org/mvndaemon/mvnd/daemon/ClientDispatcher.java @@ -38,12 +38,14 @@ import org.mvndaemon.mvnd.common.Message.BuildException; import org.mvndaemon.mvnd.common.Message.BuildStarted; import org.mvndaemon.mvnd.logging.smart.BuildEventListener; +import org.mvndaemon.mvnd.logging.smart.TestBuildSummary; /** * Sends events back to the client. */ public class ClientDispatcher extends BuildEventListener { private final Collection queue; + private final TestBuildSummary testSummary = new TestBuildSummary(); private static final Pattern TRAILING_EOLS_PATTERN = Pattern.compile("[\r\n]+$"); public ClientDispatcher(Collection queue) { @@ -130,6 +132,58 @@ public void mojoStarted(ExecutionEvent event) { execution.getExecutionId())); } + public void testProgress( + String projectId, + int forkChannelId, + String testClass, + String testMethod, + int completed, + int failures, + int errors, + int skipped, + int retrying, + int flaky, + List flakyTests, + List failedTests, + List erroredTests) { + queue.add(Message.projectTestProgress( + projectId, + forkChannelId, + testClass, + testMethod, + completed, + failures, + errors, + skipped, + retrying, + flaky, + flakyTests, + failedTests, + erroredTests)); + testSummary.record( + projectId, + forkChannelId, + completed, + failures, + errors, + skipped, + retrying, + flaky, + flakyTests, + failedTests, + erroredTests); + } + + @Override + public void foldTestProgress(String projectId) { + testSummary.foldProject(projectId); + } + + @Override + public TestBuildSummary getTestSummary() { + return testSummary; + } + public void finish(int exitCode) throws Exception { queue.add(new Message.BuildFinished(exitCode)); queue.add(Message.BareMessage.STOP_SINGLETON); diff --git a/daemon/src/main/java/org/mvndaemon/mvnd/daemon/MvndTestProgressLifecycleParticipant.java b/daemon/src/main/java/org/mvndaemon/mvnd/daemon/MvndTestProgressLifecycleParticipant.java new file mode 100644 index 000000000..527034279 --- /dev/null +++ b/daemon/src/main/java/org/mvndaemon/mvnd/daemon/MvndTestProgressLifecycleParticipant.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.daemon; + +import javax.inject.Named; +import javax.inject.Singleton; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.maven.AbstractMavenLifecycleParticipant; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.model.Plugin; +import org.apache.maven.model.PluginExecution; +import org.apache.maven.project.MavenProject; +import org.codehaus.plexus.util.xml.Xpp3Dom; +import org.mvndaemon.mvnd.common.Environment; + +/** + * Under Maven 3.9, {@code afterProjectsRead} model mutation is honored by Surefire, so this participant injects an + * mvnd {@code } into surefire {@code maven-surefire-plugin} and {@code maven-failsafe-plugin} + * configurations. Surefire then loads {@code MvndForkNodeFactory} (added to its plugin realm by + * {@link org.mvndaemon.mvnd.cache.invalidating.InvalidatingPluginRealmCache}) and reports per-test events tagged + * with the injected {@code }. Skips projects where the user already configured a {@code } or + * when the feature is disabled. + */ +@Named +@Singleton +public class MvndTestProgressLifecycleParticipant extends AbstractMavenLifecycleParticipant { + + private static final String FORK_NODE_IMPL = "org.mvndaemon.mvnd.forknode.MvndForkNodeFactory"; + private static final String SUREFIRE_KEY = "org.apache.maven.plugins:maven-surefire-plugin"; + private static final String FAILSAFE_KEY = "org.apache.maven.plugins:maven-failsafe-plugin"; + private static final Pattern VERSION_PATTERN = Pattern.compile("^(\\d+)\\.(\\d+)\\.(\\d+)(?:-M(\\d+))?"); + + @Override + public void afterProjectsRead(MavenSession session) { + if (!isTestProgressEnabled()) { + return; + } + for (MavenProject project : session.getProjects()) { + injectForPlugin(project, SUREFIRE_KEY); + injectForPlugin(project, FAILSAFE_KEY); + } + } + + private void injectForPlugin(MavenProject project, String pluginKey) { + if (project.getBuild() == null) { + return; + } + Plugin plugin = project.getBuild().getPluginsAsMap().get(pluginKey); + if (plugin == null) { + return; + } + if (!supportsForkNode(plugin.getVersion())) { + // Never inject into a Surefire/Failsafe that cannot load the fork-node SPI; it would fail the build. + return; + } + String projectId = project.getArtifactId(); + plugin.setConfiguration(withForkNode((Xpp3Dom) plugin.getConfiguration(), projectId)); + for (PluginExecution execution : plugin.getExecutions()) { + execution.setConfiguration(withForkNode((Xpp3Dom) execution.getConfiguration(), projectId)); + } + } + + private Xpp3Dom withForkNode(Xpp3Dom config, String projectId) { + if (config == null) { + config = new Xpp3Dom("configuration"); + } + if (config.getChild("forkNode") != null) { + return config; // respect a user-configured fork node + } + Xpp3Dom forkNode = new Xpp3Dom("forkNode"); + forkNode.setAttribute("implementation", FORK_NODE_IMPL); + Xpp3Dom pid = new Xpp3Dom("projectId"); + pid.setValue(projectId); + forkNode.addChild(pid); + config.addChild(forkNode); + return config; + } + + /** Shared with {@link Server}, which enables the daemon-side test progress listener under the same flag. */ + static boolean isTestProgressEnabled() { + return Environment.MVND_TEST_PROGRESS + .asOptional() + .map(Boolean::parseBoolean) + .orElse(Boolean.TRUE); + } + + /** + * The {@code forkNode} extension SPI ({@code ForkNodeFactory} / {@code SurefireForkNodeFactory}) exists only in + * Surefire {@code >= 3.0.0-M5}. Injecting {@code } into anything older makes the build fail hard + * ("unknown parameter forkNode" or a missing implementation class), so guard on the resolved plugin version. + */ + static boolean supportsForkNode(String version) { + if (version == null) { + return false; + } + Matcher m = VERSION_PATTERN.matcher(version); + if (!m.find()) { + return false; + } + int major = Integer.parseInt(m.group(1)); + if (major != 3) { + return major > 3; + } + int minor = Integer.parseInt(m.group(2)); + int patch = Integer.parseInt(m.group(3)); + if (minor != 0 || patch != 0) { + // The 3.0.0-Mx milestone series is the only pre-GA run; 3.1.0+ always shipped GA. + return true; + } + String milestone = m.group(4); + return milestone == null || Integer.parseInt(milestone) >= 5; + } +} diff --git a/daemon/src/main/java/org/mvndaemon/mvnd/daemon/Server.java b/daemon/src/main/java/org/mvndaemon/mvnd/daemon/Server.java index 819f9fbce..5e5c2c529 100644 --- a/daemon/src/main/java/org/mvndaemon/mvnd/daemon/Server.java +++ b/daemon/src/main/java/org/mvndaemon/mvnd/daemon/Server.java @@ -70,6 +70,7 @@ import org.mvndaemon.mvnd.logging.smart.BuildEventListener; import org.mvndaemon.mvnd.logging.smart.LoggingOutputStream; import org.mvndaemon.mvnd.logging.smart.ProjectBuildLogAppender; +import org.mvndaemon.mvnd.testprogress.MvndTestProgress; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -510,6 +511,36 @@ private void handle(DaemonConnection connection, BuildRequest buildRequest) { final BlockingQueue sendQueue = new PriorityBlockingQueue<>(64, Message.getMessageComparator()); final BlockingQueue recvQueue = new LinkedBlockingDeque<>(); final BuildEventListener buildEventListener = new ClientDispatcher(sendQueue); + if (MvndTestProgressLifecycleParticipant.isTestProgressEnabled()) { + final ClientDispatcher clientDispatcher = (ClientDispatcher) buildEventListener; + MvndTestProgress.setListener( + (projectId, + forkChannelId, + testClass, + testMethod, + completed, + failures, + errors, + skipped, + retrying, + flaky, + flakyTests, + failedTests, + erroredTests) -> clientDispatcher.testProgress( + projectId, + forkChannelId, + testClass, + testMethod, + completed, + failures, + errors, + skipped, + retrying, + flaky, + flakyTests, + failedTests, + erroredTests)); + } final DaemonInputStream daemonInputStream = new DaemonInputStream(projectId -> sendQueue.add(Message.requestInput(projectId))); try (ProjectBuildLogAppender logAppender = new ProjectBuildLogAppender(buildEventListener)) { @@ -643,6 +674,7 @@ public T request(Message request, Class responseType, Pre } catch (Throwable t) { LOGGER.error("Error while building project", t); } finally { + MvndTestProgress.setListener(null); if (!noDaemon) { LOGGER.info("Daemon back to idle"); updateState(DaemonState.Idle); diff --git a/daemon/src/test/java/org/mvndaemon/mvnd/daemon/MvndTestProgressLifecycleParticipantTest.java b/daemon/src/test/java/org/mvndaemon/mvnd/daemon/MvndTestProgressLifecycleParticipantTest.java new file mode 100644 index 000000000..7643e22db --- /dev/null +++ b/daemon/src/test/java/org/mvndaemon/mvnd/daemon/MvndTestProgressLifecycleParticipantTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.daemon; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class MvndTestProgressLifecycleParticipantTest { + + @Test + void injectsOnlyForSurefireVersionsThatSupportForkNode() { + // forkNode SPI exists since 3.0.0-M5 -> anything older must be skipped so the build never fails + assertFalse(MvndTestProgressLifecycleParticipant.supportsForkNode(null)); + assertFalse(MvndTestProgressLifecycleParticipant.supportsForkNode("2.22.2")); + assertFalse(MvndTestProgressLifecycleParticipant.supportsForkNode("3.0.0-M4")); + + assertTrue(MvndTestProgressLifecycleParticipant.supportsForkNode("3.0.0-M5")); + assertTrue(MvndTestProgressLifecycleParticipant.supportsForkNode("3.0.0-M8")); + assertTrue(MvndTestProgressLifecycleParticipant.supportsForkNode("3.5.6")); + assertTrue(MvndTestProgressLifecycleParticipant.supportsForkNode("4.0.0")); + } + + @Test + void milestoneGuardOnlyAppliesToThe300MilestoneSeries() { + // only 3.0.0-Mx predates the forkNode SPI; 3.1.0+ always shipped GA, so an "-Mx" suffix there + // must not be mistaken for a pre-SPI milestone build. + assertTrue(MvndTestProgressLifecycleParticipant.supportsForkNode("3.1.0-M2")); + assertTrue(MvndTestProgressLifecycleParticipant.supportsForkNode("3.2.5-M1")); + } +} diff --git a/dist/src/main/distro/bin/mvnd-bash-completion.bash b/dist/src/main/distro/bin/mvnd-bash-completion.bash index 692534a07..648030e4f 100755 --- a/dist/src/main/distro/bin/mvnd-bash-completion.bash +++ b/dist/src/main/distro/bin/mvnd-bash-completion.bash @@ -218,7 +218,7 @@ _mvnd() local mvnd_opts="-1" local mvnd_long_opts="--color|--completion|--diag|--purge|--raw-streams|--serial|--status|--stop" - local mvnd_properties="-Djava.home|-Djdk.java.options|-Dmaven.multiModuleProjectDirectory|-Dmaven.repo.local|-Dmaven.settings|-Dmvnd.buildTime|-Dmvnd.builder|-Dmvnd.cancelConnectTimeout|-Dmvnd.connectTimeout|-Dmvnd.coreExtensionsExclude|-Dmvnd.daemonStorage|-Dmvnd.debug|-Dmvnd.debug.address|-Dmvnd.duplicateDaemonGracePeriod|-Dmvnd.enableAssertions|-Dmvnd.expirationCheckDelay|-Dmvnd.home|-Dmvnd.idleTimeout|-Dmvnd.jvmArgs|-Dmvnd.keepAlive|-Dmvnd.logPurgePeriod|-Dmvnd.maxHeapSize|-Dmvnd.maxLostKeepAlive|-Dmvnd.minHeapSize|-Dmvnd.minThreads|-Dmvnd.noBuffering|-Dmvnd.noDaemon|-Dmvnd.noModelCache|-Dmvnd.pluginRealmEvictPattern|-Dmvnd.propertiesPath|-Dmvnd.rawStreams|-Dmvnd.registry|-Dmvnd.rollingWindowSize|-Dmvnd.serial|-Dmvnd.socketConnectTimeout|-Dmvnd.socketFamily|-Dmvnd.threadStackSize|-Dmvnd.threads|-Dstyle.color|-Duser.dir|-Duser.home" + local mvnd_properties="-Djava.home|-Djdk.java.options|-Dmaven.multiModuleProjectDirectory|-Dmaven.repo.local|-Dmaven.settings|-Dmvnd.buildTime|-Dmvnd.builder|-Dmvnd.cancelConnectTimeout|-Dmvnd.connectTimeout|-Dmvnd.coreExtensionsExclude|-Dmvnd.daemonStorage|-Dmvnd.debug|-Dmvnd.debug.address|-Dmvnd.duplicateDaemonGracePeriod|-Dmvnd.enableAssertions|-Dmvnd.expirationCheckDelay|-Dmvnd.hideBannedProjectSkips|-Dmvnd.home|-Dmvnd.idleTimeout|-Dmvnd.jvmArgs|-Dmvnd.keepAlive|-Dmvnd.logPurgePeriod|-Dmvnd.maxHeapSize|-Dmvnd.maxLostKeepAlive|-Dmvnd.minHeapSize|-Dmvnd.minThreads|-Dmvnd.noBuffering|-Dmvnd.noDaemon|-Dmvnd.noModelCache|-Dmvnd.pluginRealmEvictPattern|-Dmvnd.propertiesPath|-Dmvnd.rawStreams|-Dmvnd.registry|-Dmvnd.rollingWindowSize|-Dmvnd.serial|-Dmvnd.socketConnectTimeout|-Dmvnd.socketFamily|-Dmvnd.testProgress|-Dmvnd.threadStackSize|-Dmvnd.threads|-Dstyle.color|-Duser.dir|-Duser.home" local opts="-am|-amd|-B|-C|-c|-cpu|-D|-e|-emp|-ep|-f|-fae|-ff|-fn|-gs|-h|-l|-N|-npr|-npu|-nsu|-o|-P|-pl|-q|-rf|-s|-T|-t|-U|-up|-V|-v|-X|${mvnd_opts}" local long_opts="--also-make|--also-make-dependents|--batch-mode|--strict-checksums|--lax-checksums|--check-plugin-updates|--define|--errors|--encrypt-master-password|--encrypt-password|--file|--fail-at-end|--fail-fast|--fail-never|--global-settings|--help|--log-file|--non-recursive|--no-plugin-registry|--no-plugin-updates|--no-snapshot-updates|--offline|--activate-profiles|--projects|--quiet|--resume-from|--settings|--threads|--toolchains|--update-snapshots|--update-plugins|--show-version|--version|--debug|${mvnd_long_opts}" diff --git a/dist/src/main/provisio/maven-distro.xml b/dist/src/main/provisio/maven-distro.xml index e7ff5d465..618c4c74b 100644 --- a/dist/src/main/provisio/maven-distro.xml +++ b/dist/src/main/provisio/maven-distro.xml @@ -46,6 +46,9 @@ + + + diff --git a/integration-tests/src/test/java/org/mvndaemon/mvnd/it/InteractiveTest.java b/integration-tests/src/test/java/org/mvndaemon/mvnd/it/InteractiveTest.java index 46ea1afab..ccfd51ed6 100644 --- a/integration-tests/src/test/java/org/mvndaemon/mvnd/it/InteractiveTest.java +++ b/integration-tests/src/test/java/org/mvndaemon/mvnd/it/InteractiveTest.java @@ -21,6 +21,7 @@ import javax.inject.Inject; import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -59,10 +60,12 @@ void versionsSet() throws IOException, InterruptedException { .findFirst() .get(); + final AtomicBoolean promptSeen = new AtomicBoolean(); final TestClientOutput o = new TestClientOutput() { @Override public void accept(Message m) { if (m instanceof Prompt) { + promptSeen.set(true); daemonDispatch.accept(((Prompt) m).response("0.1.0-SNAPSHOT")); } super.accept(m); @@ -73,6 +76,7 @@ public void accept(Message m) { } else { client.execute(o, "versions:set").assertSuccess(); } + Assertions.assertTrue(promptSeen.get(), "versions:set must request the new version"); final String newVersion = MvndTestUtil.version(parameters.multiModuleProjectDirectory().resolve("pom.xml")); diff --git a/integration-tests/src/test/java/org/mvndaemon/mvnd/it/TestProgressFailureTest.java b/integration-tests/src/test/java/org/mvndaemon/mvnd/it/TestProgressFailureTest.java new file mode 100644 index 000000000..5325dd538 --- /dev/null +++ b/integration-tests/src/test/java/org/mvndaemon/mvnd/it/TestProgressFailureTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.it; + +import javax.inject.Inject; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.mvndaemon.mvnd.assertj.TestClientOutput; +import org.mvndaemon.mvnd.client.Client; +import org.mvndaemon.mvnd.common.Message; +import org.mvndaemon.mvnd.junit.MvndTest; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +@MvndTest(projectDir = "src/test/projects/test-progress-failure") +class TestProgressFailureTest { + + @Inject + Client client; + + @Test + void reportsFailedAndErroredTestsWithMessages() throws InterruptedException { + final TestClientOutput output = new TestClientOutput(); + client.execute(output, "clean", "test", "-B").assertFailure(); + + List events = output.getMessages().stream() + .filter(Message.ProjectTestProgressEvent.class::isInstance) + .map(Message.ProjectTestProgressEvent.class::cast) + .toList(); + + assertTrue(!events.isEmpty(), "expected PROJECT_TEST_PROGRESS messages, got none"); + assertTrue( + events.stream() + .flatMap(e -> e.getFailedTests().stream()) + .anyMatch(t -> t.startsWith("FailingServiceTest#failsAssertion") && t.contains(": ")), + "expected the failed test to be reported with its message"); + assertTrue( + events.stream() + .flatMap(e -> e.getErroredTests().stream()) + .anyMatch(t -> t.startsWith("FailingServiceTest#throwsError") && t.contains(": ")), + "expected the errored test to be reported with its message"); + + List logLines = output.getMessages().stream() + .filter(Message.StringMessage.class::isInstance) + .filter(m -> m.getType() == Message.BUILD_LOG_MESSAGE) + .map(Message.StringMessage.class::cast) + .map(Message.StringMessage::getMessage) + .toList(); + + int failuresLine = indexOfLineContaining(logLines, "Failures:"); + int errorsLine = indexOfLineContaining(logLines, "Errors:"); + // test-progress-failure is a single-module fixture, so Maven never prints a "Reactor Summary" section; + // "BUILD FAILURE" is the banner that is always emitted, so it is the anchor used here instead. + int buildFailureLine = indexOfLineContaining(logLines, "BUILD FAILURE"); + assertTrue(failuresLine >= 0, "expected a daemon-emitted log line containing 'Failures:', got: " + logLines); + assertTrue(errorsLine >= 0, "expected a daemon-emitted log line containing 'Errors:', got: " + logLines); + assertTrue(buildFailureLine >= 0, "expected a 'BUILD FAILURE' log line, got: " + logLines); + assertTrue(failuresLine < buildFailureLine, "the test summary must be logged before the BUILD FAILURE banner"); + assertTrue(errorsLine < buildFailureLine, "the test summary must be logged before the BUILD FAILURE banner"); + } + + private static int indexOfLineContaining(List lines, String needle) { + for (int i = 0; i < lines.size(); i++) { + if (lines.get(i).contains(needle)) { + return i; + } + } + return -1; + } +} diff --git a/integration-tests/src/test/java/org/mvndaemon/mvnd/it/TestProgressTest.java b/integration-tests/src/test/java/org/mvndaemon/mvnd/it/TestProgressTest.java new file mode 100644 index 000000000..4fae7aa7b --- /dev/null +++ b/integration-tests/src/test/java/org/mvndaemon/mvnd/it/TestProgressTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.it; + +import javax.inject.Inject; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.mvndaemon.mvnd.assertj.TestClientOutput; +import org.mvndaemon.mvnd.client.Client; +import org.mvndaemon.mvnd.common.Message; +import org.mvndaemon.mvnd.junit.MvndTest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@MvndTest(projectDir = "src/test/projects/test-progress") +class TestProgressTest { + + @Inject + Client client; + + @Test + void emitsIncreasingTestProgress() throws InterruptedException { + final TestClientOutput output = new TestClientOutput(); + client.execute(output, "clean", "test", "-B").assertSuccess(); + + List events = testProgressEvents(output); + + assertTrue(!events.isEmpty(), "expected PROJECT_TEST_PROGRESS messages, got none"); + int maxCompleted = events.stream() + .mapToInt(Message.ProjectTestProgressEvent::getCompleted) + .max() + .orElse(0); + assertTrue( + maxCompleted >= 3, + "expected completed count to reach the number of executed tests, got " + maxCompleted); + assertTrue( + events.stream().anyMatch(e -> "org.mvndaemon.mvnd.test.MyServiceTest".equals(e.getTestClass())), + "expected the current test class name to be reported"); + } + + @Test + void emitsFlakyTestProgress() throws InterruptedException { + final TestClientOutput output = new TestClientOutput(); + client.execute(output, "clean", "test", "-B").assertSuccess(); + + List events = testProgressEvents(output); + + assertTrue( + events.stream().anyMatch(e -> e.getRetrying() > 0), + "expected a retrying snapshot while the flaky test was being rerun"); + assertTrue( + events.stream().anyMatch(e -> e.getFlaky() > 0), "expected a flaky snapshot after the rerun succeeded"); + assertTrue( + events.stream() + .flatMap(e -> e.getFlakyTests().stream()) + .anyMatch(t -> t.startsWith("FlakyServiceTest#succeedsOnRetry")), + "expected the recovered test to be reported in the flaky test list"); + } + + @Test + void disabledEmitsNoTestProgress() throws InterruptedException { + final TestClientOutput output = new TestClientOutput(); + client.execute(output, "clean", "test", "-B", "-Dmvnd.testProgress=false") + .assertSuccess(); + + assertEquals(0, testProgressEvents(output).size(), "no test-progress messages expected when feature disabled"); + } + + private static List testProgressEvents(TestClientOutput output) { + return output.getMessages().stream() + .filter(Message.ProjectTestProgressEvent.class::isInstance) + .map(Message.ProjectTestProgressEvent.class::cast) + .toList(); + } +} diff --git a/integration-tests/src/test/projects/test-progress-failure/pom.xml b/integration-tests/src/test/projects/test-progress-failure/pom.xml new file mode 100644 index 000000000..d075a6996 --- /dev/null +++ b/integration-tests/src/test/projects/test-progress-failure/pom.xml @@ -0,0 +1,59 @@ + + + + 4.0.0 + org.mvndaemon.mvnd.test.test-progress-failure + test-progress-failure + 0.0.1-SNAPSHOT + jar + + + UTF-8 + 17 + 17 + + 3.5.6 + 5.14.4 + + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + false + + + + + + diff --git a/integration-tests/src/test/projects/test-progress-failure/src/test/java/org/mvndaemon/mvnd/test/FailingServiceTest.java b/integration-tests/src/test/projects/test-progress-failure/src/test/java/org/mvndaemon/mvnd/test/FailingServiceTest.java new file mode 100644 index 000000000..9fde3e152 --- /dev/null +++ b/integration-tests/src/test/projects/test-progress-failure/src/test/java/org/mvndaemon/mvnd/test/FailingServiceTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.test; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FailingServiceTest { + + @Test + void failsAssertion() { + assertEquals(5, 2 + 2, "arithmetic is broken"); + } + + @Test + void throwsError() { + throw new IllegalStateException("service unavailable"); + } +} diff --git a/integration-tests/src/test/projects/test-progress/pom.xml b/integration-tests/src/test/projects/test-progress/pom.xml new file mode 100644 index 000000000..9b2156093 --- /dev/null +++ b/integration-tests/src/test/projects/test-progress/pom.xml @@ -0,0 +1,58 @@ + + + + 4.0.0 + org.mvndaemon.mvnd.test.test-progress + test-progress + 0.0.1-SNAPSHOT + jar + + + UTF-8 + 17 + 17 + + 3.5.6 + 5.14.4 + + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + 1 + + + + + + diff --git a/integration-tests/src/test/projects/test-progress/src/test/java/org/mvndaemon/mvnd/test/FlakyServiceTest.java b/integration-tests/src/test/projects/test-progress/src/test/java/org/mvndaemon/mvnd/test/FlakyServiceTest.java new file mode 100644 index 000000000..183be43de --- /dev/null +++ b/integration-tests/src/test/projects/test-progress/src/test/java/org/mvndaemon/mvnd/test/FlakyServiceTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.test; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FlakyServiceTest { + + private static final AtomicInteger attempts = new AtomicInteger(); + + @Test + void succeedsOnRetry() { + assertTrue(attempts.incrementAndGet() >= 2, "first attempt fails, rerun should pass"); + } +} diff --git a/integration-tests/src/test/projects/test-progress/src/test/java/org/mvndaemon/mvnd/test/MyServiceTest.java b/integration-tests/src/test/projects/test-progress/src/test/java/org/mvndaemon/mvnd/test/MyServiceTest.java new file mode 100644 index 000000000..d3e0a78e6 --- /dev/null +++ b/integration-tests/src/test/projects/test-progress/src/test/java/org/mvndaemon/mvnd/test/MyServiceTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.test; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class MyServiceTest { + + @Test + void shouldWork() { + assertEquals(2, 1 + 1); + } + + @Test + void alsoWorks() { + assertEquals(4, 2 + 2); + } + + @Test + void andAgain() { + assertEquals(9, 3 * 3); + } + + @Test + @Disabled("intentionally skipped to exercise the skipped count") + void skippedForNow() { + assertEquals(1, 2); + } +} diff --git a/logging/pom.xml b/logging/pom.xml index fa121e2bc..91742d064 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -53,6 +53,12 @@ org.slf4j jul-to-slf4j + + + org.junit.jupiter + junit-jupiter + test + diff --git a/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/BuildEventListener.java b/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/BuildEventListener.java index be905d27a..2e4fc057c 100644 --- a/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/BuildEventListener.java +++ b/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/BuildEventListener.java @@ -76,4 +76,12 @@ protected BuildEventListener() {} public abstract void log(String msg); public abstract void transfer(String projectId, TransferEvent e); + + /** Folds the given project's in-flight test-progress snapshots into the running reactor-wide totals. */ + public void foldTestProgress(String projectId) {} + + /** @return the collected test-summary data, or {@code null} if this listener does not track one */ + public TestBuildSummary getTestSummary() { + return null; + } } diff --git a/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/LoggingExecutionListener.java b/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/LoggingExecutionListener.java index eccd2ba78..039455f0f 100644 --- a/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/LoggingExecutionListener.java +++ b/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/LoggingExecutionListener.java @@ -21,6 +21,8 @@ import javax.inject.Named; import javax.inject.Singleton; +import java.util.List; + import org.apache.maven.execution.ExecutionEvent; import org.apache.maven.execution.ExecutionListener; import org.apache.maven.execution.MavenExecutionRequest; @@ -30,12 +32,18 @@ import org.apache.maven.lifecycle.LifecycleExecutionException; import org.apache.maven.lifecycle.internal.ReactorBuildStatus; import org.eclipse.sisu.Typed; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Singleton @Named @Typed({LoggingExecutionListener.class, ExecutionListener.class, ProjectExecutionListener.class}) public class LoggingExecutionListener implements ExecutionListener, ProjectExecutionListener { + /** Binds to {@code MvndSimpleLogger} because this class lives in the Maven realm, so the summary is colorized + * and routed to the client through the same pipeline as any other Maven console line. */ + private static final Logger LOGGER = LoggerFactory.getLogger("org.mvndaemon.mvnd.testsummary"); + private ExecutionListener delegate; private BuildEventListener buildEventListener; @@ -93,9 +101,40 @@ public void sessionStarted(ExecutionEvent event) { @Override public void sessionEnded(ExecutionEvent event) { setMdc(event); + emitTestSummary(); delegate.sessionEnded(event); } + /** + * Logs the collected test summary immediately before Maven's Reactor Summary/BUILD banner (emitted next by + * {@code delegate.sessionEnded}), through this listener's own {@link #LOGGER} so coloring, the {@code [LEVEL]} + * prefix, and {@code -q} gating all come from the normal Maven logging pipeline. + */ + private void emitTestSummary() { + TestBuildSummary summary = buildEventListener.getTestSummary(); + if (summary == null) { + return; + } + List lines = summary.renderLines(); + if (lines.isEmpty()) { + return; + } + ProjectBuildLogAppender.setProjectId(null); + for (TestBuildSummary.SummaryLine line : lines) { + switch (line.level) { + case ERROR: + LOGGER.error(line.text); + break; + case WARNING: + LOGGER.warn(line.text); + break; + default: + LOGGER.info(line.text); + break; + } + } + } + @Override public void projectStarted(ExecutionEvent event) { setMdc(event); @@ -130,6 +169,8 @@ public void mojoStarted(ExecutionEvent event) { setMdc(event); buildEventListener.mojoStarted(event); delegate.mojoStarted(event); + // Folds the previous test-running mojo's snapshots into the reactor totals; a no-op for non-test mojos. + buildEventListener.foldTestProgress(event.getProject().getArtifactId()); } @Override diff --git a/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/TestBuildSummary.java b/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/TestBuildSummary.java new file mode 100644 index 000000000..9cdfba837 --- /dev/null +++ b/logging/src/main/java/org/mvndaemon/mvnd/logging/smart/TestBuildSummary.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.logging.smart; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Collects the reactor-wide failed/errored/flaky test identities and numeric totals needed to render the + * end-of-build test summary. Records arrive on fork-reader threads via {@link #record}, are folded into the + * running totals on build threads via {@link #foldProject}, and are rendered on the main thread via + * {@link #renderLines()}; all three are synchronized so the object can be shared across those threads. + */ +public class TestBuildSummary { + + /** Test-summary line severity, mirroring Maven's INFO/WARNING/ERROR levels. */ + public enum SummaryLevel { + INFO, + WARNING, + ERROR + } + + /** One line of the rendered summary, tagged with the level it should be logged at. */ + public static final class SummaryLine { + public final SummaryLevel level; + public final String text; + + SummaryLine(SummaryLevel level, String text) { + this.level = level; + this.text = text; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof SummaryLine)) { + return false; + } + SummaryLine other = (SummaryLine) o; + return level == other.level && text.equals(other.text); + } + + @Override + public int hashCode() { + return 31 * level.hashCode() + text.hashCode(); + } + + @Override + public String toString() { + return "[" + level + "] " + text; + } + } + + private final Map> failedTests = new LinkedHashMap<>(); + private final Map> erroredTests = new LinkedHashMap<>(); + private final Map> flakyTests = new LinkedHashMap<>(); + /** Latest cumulative per-fork snapshot for each project, indexed by fork channel id. */ + private final Map> currentByProject = new LinkedHashMap<>(); + + /** Indices into the per-fork snapshot {@code int[]} recorded by {@link #record} and folded by {@link #foldProject}. */ + private static final int IDX_COMPLETED = 0; + + private static final int IDX_FAILURES = 1; + private static final int IDX_ERRORS = 2; + private static final int IDX_SKIPPED = 3; + // IDX_RETRYING = 4 is intentionally not folded into totals: a test still retrying has no final outcome yet. + private static final int IDX_FLAKY = 5; + + private TestTotals totals = TestTotals.EMPTY; + + /** + * Records one project/fork's latest cumulative snapshot and unions in any newly reported test identities. + * Union is collision-free, so fork-channel-id reuse across surefire/failsafe does not affect the identity sets. + */ + public synchronized void record( + String projectId, + int forkChannelId, + int completed, + int failures, + int errors, + int skipped, + int retrying, + int flaky, + List flakyTests, + List failedTests, + List erroredTests) { + if (!flakyTests.isEmpty()) { + this.flakyTests + .computeIfAbsent(projectId, k -> new LinkedHashSet<>()) + .addAll(flakyTests); + } + if (!failedTests.isEmpty()) { + this.failedTests + .computeIfAbsent(projectId, k -> new LinkedHashSet<>()) + .addAll(failedTests); + } + if (!erroredTests.isEmpty()) { + this.erroredTests + .computeIfAbsent(projectId, k -> new LinkedHashSet<>()) + .addAll(erroredTests); + } + currentByProject.computeIfAbsent(projectId, k -> new LinkedHashMap<>()).put(forkChannelId, new int[] { + completed, failures, errors, skipped, retrying, flaky + }); // indices: see IDX_* fields + } + + /** + * Sums the given project's latest per-fork snapshots into the running reactor-wide totals, then clears them. + * Called at each mojo boundary so cumulative-per-fork counts are summed correctly across surefire+failsafe + * (fork ids restart per plugin execution). A no-op if the project reported no test progress. + */ + public synchronized void foldProject(String projectId) { + Map snapshots = currentByProject.remove(projectId); + if (snapshots == null || snapshots.isEmpty()) { + return; + } + int completed = 0; + int failures = 0; + int errors = 0; + int skipped = 0; + int flaky = 0; + for (int[] snapshot : snapshots.values()) { + completed += snapshot[IDX_COMPLETED]; + failures += snapshot[IDX_FAILURES]; + errors += snapshot[IDX_ERRORS]; + skipped += snapshot[IDX_SKIPPED]; + flaky += snapshot[IDX_FLAKY]; + } + totals = new TestTotals( + totals.completed + completed, + totals.failures + failures, + totals.errors + errors, + totals.skipped + skipped, + totals.flaky + flaky); + } + + /** + * Folds in any projects whose snapshots have not yet been folded, then renders the reactor-wide failed/errored/ + * flaky test summary in the same shape Surefire itself uses (Results: / Failures: / Errors: / Flakes: / + * Tests run: ...). Returns an empty list when there is nothing to report. + */ + public synchronized List renderLines() { + for (String projectId : new ArrayList<>(currentByProject.keySet())) { + foldProject(projectId); + } + List out = new ArrayList<>(); + if (failedTests.isEmpty() && erroredTests.isEmpty() && flakyTests.isEmpty()) { + return out; + } + emit(out, SummaryLevel.INFO, ""); + emit(out, SummaryLevel.INFO, "Results:"); + emit(out, SummaryLevel.INFO, ""); + emitFailureCategory(out, "Failures: ", failedTests); + emitFailureCategory(out, "Errors: ", erroredTests); + emitFlakyCategory(out, "Flakes: ", flakyTests); + emit(out, SummaryLevel.INFO, ""); + emit(out, trailerLevel(totals), trailerLine(totals)); + emit(out, SummaryLevel.INFO, ""); + return out; + } + + private static void emit(List out, SummaryLevel level, String text) { + out.add(new SummaryLine(level, text)); + } + + /** Renders a "Failures: "/"Errors: " section: header plus one {@code " "} line per entry. */ + private static void emitFailureCategory(List out, String header, Map> byProject) { + if (byProject.isEmpty()) { + return; + } + emit(out, SummaryLevel.ERROR, header); + for (Map.Entry> entry : byProject.entrySet()) { + for (String test : entry.getValue()) { + emit(out, SummaryLevel.ERROR, " " + entry.getKey() + " " + test); + } + } + } + + /** + * Renders the "Flakes: " section. Each entry is a {@code TestProgressAccumulator}-formatted multi-line block + * (display name, then one {@code " Run N: PASS"}/{@code " Run N: "} line per attempt); this splits + * that block and re-levels each line: the header/test-name lines are WARNING, a passing run is INFO, a failing + * run is ERROR -- matching Surefire's own per-line coloring exactly. + */ + private static void emitFlakyCategory(List out, String header, Map> byProject) { + if (byProject.isEmpty()) { + return; + } + emit(out, SummaryLevel.WARNING, header); + for (Map.Entry> entry : byProject.entrySet()) { + for (String detail : entry.getValue()) { + String[] lines = detail.split("\n", -1); + emit(out, SummaryLevel.WARNING, " " + entry.getKey() + " " + lines[0]); + for (int i = 1; i < lines.length; i++) { + String runLine = lines[i]; + boolean passed = runLine.trim().endsWith(": PASS"); + emit(out, passed ? SummaryLevel.INFO : SummaryLevel.ERROR, " " + runLine); + } + } + } + } + + private static SummaryLevel trailerLevel(TestTotals totals) { + if (totals.failures > 0 || totals.errors > 0) { + return SummaryLevel.ERROR; + } + return totals.flaky > 0 ? SummaryLevel.WARNING : SummaryLevel.INFO; + } + + private static String trailerLine(TestTotals totals) { + StringBuilder sb = new StringBuilder("Tests run: ") + .append(totals.completed) + .append(", Failures: ") + .append(totals.failures) + .append(", Errors: ") + .append(totals.errors) + .append(", Skipped: ") + .append(totals.skipped); + if (totals.flaky > 0) { + sb.append(", Flakes: ").append(totals.flaky); + } + return sb.toString(); + } + + /** Reactor-wide test totals, folded in as each project's test-running mojo execution finishes. */ + static final class TestTotals { + static final TestTotals EMPTY = new TestTotals(0, 0, 0, 0, 0); + + final int completed; + final int failures; + final int errors; + final int skipped; + final int flaky; + + TestTotals(int completed, int failures, int errors, int skipped, int flaky) { + this.completed = completed; + this.failures = failures; + this.errors = errors; + this.skipped = skipped; + this.flaky = flaky; + } + } +} diff --git a/logging/src/main/java/org/slf4j/impl/MvndSimpleLogger.java b/logging/src/main/java/org/slf4j/impl/MvndSimpleLogger.java index e8325c826..b44630519 100644 --- a/logging/src/main/java/org/slf4j/impl/MvndSimpleLogger.java +++ b/logging/src/main/java/org/slf4j/impl/MvndSimpleLogger.java @@ -134,7 +134,7 @@ protected void doLog(int level, String message, Throwable t) { if (sink != null) { sink.accept(buf.toString()); } else { - CONFIG_PARAMS.outputChoice.getTargetPrintStream().println(buf.toString()); + CONFIG_PARAMS.outputChoice.getTargetPrintStream().println(buf); } } @@ -150,16 +150,16 @@ private String computeShortName() { protected String renderLevel(int level) { switch (level) { case LOG_LEVEL_TRACE: - return level().debug("TRACE").toString(); + return level().debug("TRACE"); case LOG_LEVEL_DEBUG: - return level().debug("DEBUG").toString(); + return level().debug("DEBUG"); case LOG_LEVEL_INFO: - return level().info("INFO").toString(); + return level().info("INFO"); case LOG_LEVEL_WARN: - return level().warning("WARNING").toString(); + return level().warning("WARNING"); case LOG_LEVEL_ERROR: default: - return level().error("ERROR").toString(); + return level().error("ERROR"); } } diff --git a/logging/src/test/java/org/mvndaemon/mvnd/logging/smart/TestBuildSummaryTest.java b/logging/src/test/java/org/mvndaemon/mvnd/logging/smart/TestBuildSummaryTest.java new file mode 100644 index 000000000..7bd9b3810 --- /dev/null +++ b/logging/src/test/java/org/mvndaemon/mvnd/logging/smart/TestBuildSummaryTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.logging.smart; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.mvndaemon.mvnd.logging.smart.TestBuildSummary.SummaryLevel; +import org.mvndaemon.mvnd.logging.smart.TestBuildSummary.SummaryLine; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestBuildSummaryTest { + + @Test + void rendersSurefireStyleBlockWithTagsAndTrailer() { + TestBuildSummary summary = new TestBuildSummary(); + summary.record( + "camel-jms", + 1, + 40, + 1, + 0, + 0, + 0, + 0, + List.of(), + List.of("FooTest#bar: expected <5> but was <4>"), + List.of()); + summary.foldProject("camel-jms"); + summary.record("camel-nats", 1, 1, 0, 1, 0, 0, 0, List.of(), List.of(), List.of("NatsIT#connects: refused")); + summary.foldProject("camel-nats"); + summary.record( + "camel-mllp", + 1, + 1, + 0, + 0, + 0, + 0, + 1, + List.of("FlakyTest#retries\n Run 1: boom\n Run 2: PASS"), + List.of(), + List.of()); + summary.foldProject("camel-mllp"); + + List lines = summary.renderLines(); + + assertEquals( + List.of( + line(SummaryLevel.INFO, ""), + line(SummaryLevel.INFO, "Results:"), + line(SummaryLevel.INFO, ""), + line(SummaryLevel.ERROR, "Failures: "), + line(SummaryLevel.ERROR, " camel-jms FooTest#bar: expected <5> but was <4>"), + line(SummaryLevel.ERROR, "Errors: "), + line(SummaryLevel.ERROR, " camel-nats NatsIT#connects: refused"), + line(SummaryLevel.WARNING, "Flakes: "), + line(SummaryLevel.WARNING, " camel-mllp FlakyTest#retries"), + line(SummaryLevel.ERROR, " Run 1: boom"), + line(SummaryLevel.INFO, " Run 2: PASS"), + line(SummaryLevel.INFO, ""), + line(SummaryLevel.ERROR, "Tests run: 42, Failures: 1, Errors: 1, Skipped: 0, Flakes: 1"), + line(SummaryLevel.INFO, "")), + lines); + } + + @Test + void omitsFlakesSuffixWhenNoFlakyTests() { + TestBuildSummary summary = new TestBuildSummary(); + summary.record( + "camel-jms", + 1, + 10, + 1, + 0, + 0, + 0, + 0, + List.of(), + List.of("FooTest#bar: expected <5> but was <4>"), + List.of()); + summary.foldProject("camel-jms"); + + List lines = summary.renderLines(); + + SummaryLine trailer = lines.get(lines.size() - 2); + assertEquals(SummaryLevel.ERROR, trailer.level); + assertEquals("Tests run: 10, Failures: 1, Errors: 0, Skipped: 0", trailer.text); + } + + @Test + void emitsNothingWhenAllCategoriesEmpty() { + TestBuildSummary summary = new TestBuildSummary(); + summary.record("camel-core", 1, 5, 0, 0, 0, 0, 0, List.of(), List.of(), List.of()); + summary.foldProject("camel-core"); + + assertEquals(List.of(), summary.renderLines()); + } + + @Test + void unionsFailedTestIdentitiesAcrossForksWithinTheSameProject() { + TestBuildSummary summary = new TestBuildSummary(); + summary.record("camel-jms", 1, 1, 1, 0, 0, 0, 0, List.of(), List.of("FooTest#bar: boom"), List.of()); + // A second fork on the same project reports a different failed test; identities must union, not overwrite. + summary.record("camel-jms", 2, 1, 1, 0, 0, 0, 0, List.of(), List.of("BarTest#baz: boom"), List.of()); + // Re-recording the same test on the same fork must not create a duplicate line (Set semantics). + summary.record("camel-jms", 1, 2, 1, 0, 0, 0, 0, List.of(), List.of("FooTest#bar: boom"), List.of()); + summary.foldProject("camel-jms"); + + List lines = summary.renderLines(); + List failureLines = lines.stream() + .filter(l -> l.text.startsWith(" camel-jms")) + .map(l -> l.text) + .toList(); + assertEquals(List.of(" camel-jms FooTest#bar: boom", " camel-jms BarTest#baz: boom"), failureLines); + } + + @Test + void foldProjectSumsCumulativeCountsAcrossSurefireThenFailsafeReusingForkChannelIds() { + TestBuildSummary summary = new TestBuildSummary(); + // Surefire execution: fork channel 1 finishes with 5 completed tests. + summary.record("app", 1, 5, 0, 0, 0, 0, 0, List.of(), List.of(), List.of()); + summary.foldProject("app"); + // Failsafe execution reuses fork channel id 1 with its own cumulative count; must add, not replace. + summary.record("app", 1, 3, 1, 0, 0, 0, 0, List.of(), List.of("ItTest#works: boom"), List.of()); + summary.foldProject("app"); + + List lines = summary.renderLines(); + SummaryLine trailer = lines.get(lines.size() - 2); + assertEquals("Tests run: 8, Failures: 1, Errors: 0, Skipped: 0", trailer.text); + } + + @Test + void renderLinesFoldsAnyProjectsNotYetFolded() { + TestBuildSummary summary = new TestBuildSummary(); + summary.record("app", 1, 4, 1, 0, 0, 0, 0, List.of(), List.of("FooTest#bar: boom"), List.of()); + // No explicit foldProject call: renderLines() must fold remaining snapshots itself. + + List lines = summary.renderLines(); + + assertTrue(lines.stream().anyMatch(l -> l.text.equals("Tests run: 4, Failures: 1, Errors: 0, Skipped: 0"))); + } + + private static SummaryLine line(SummaryLevel level, String text) { + return new SummaryLine(level, text); + } +} diff --git a/pom.xml b/pom.xml index 498c006aa..10cb5393d 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ agent helper common + surefire-progress client logging daemon @@ -250,6 +251,11 @@ mvnd-common ${project.version} + + org.apache.maven.daemon + mvnd-surefire-progress + ${project.version} + org.apache.maven.daemon mvnd-dist diff --git a/surefire-progress/pom.xml b/surefire-progress/pom.xml new file mode 100644 index 000000000..4bc79e9e9 --- /dev/null +++ b/surefire-progress/pom.xml @@ -0,0 +1,74 @@ + + + + + 4.0.0 + + org.apache.maven.daemon + mvnd + 1.0.7-SNAPSHOT + + + mvnd-surefire-progress + jar + Maven Daemon - Surefire Test Progress + + + 3.5.2 + + + + + org.apache.maven.daemon + mvnd-common + ${project.version} + provided + + + org.apache.maven.surefire + surefire-extensions-api + ${surefire.spi.version} + provided + + + + org.apache.maven.surefire + maven-surefire-common + ${surefire.spi.version} + provided + + + org.apache.maven.surefire + surefire-extensions-spi + ${surefire.spi.version} + provided + + + org.apache.maven.surefire + surefire-api + ${surefire.spi.version} + provided + + + org.junit.jupiter + junit-jupiter + test + + + diff --git a/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/MvndForkNodeFactory.java b/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/MvndForkNodeFactory.java new file mode 100644 index 000000000..bb12be7b8 --- /dev/null +++ b/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/MvndForkNodeFactory.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.forknode; + +import java.io.IOException; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +import org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory; +import org.apache.maven.surefire.api.event.Event; +import org.apache.maven.surefire.api.event.TestErrorEvent; +import org.apache.maven.surefire.api.event.TestFailedEvent; +import org.apache.maven.surefire.api.event.TestSkippedEvent; +import org.apache.maven.surefire.api.event.TestStartingEvent; +import org.apache.maven.surefire.api.event.TestSucceededEvent; +import org.apache.maven.surefire.api.event.TestsetCompletedEvent; +import org.apache.maven.surefire.api.event.TestsetStartingEvent; +import org.apache.maven.surefire.api.fork.ForkNodeArguments; +import org.apache.maven.surefire.api.report.ReportEntry; +import org.apache.maven.surefire.api.report.SafeThrowable; +import org.apache.maven.surefire.api.report.StackTraceWriter; +import org.apache.maven.surefire.extensions.CommandReader; +import org.apache.maven.surefire.extensions.EventHandler; +import org.apache.maven.surefire.extensions.ForkChannel; +import org.apache.maven.surefire.extensions.util.CountdownCloseable; +import org.mvndaemon.mvnd.testprogress.MvndTestProgress; + +/** + * A {@link org.apache.maven.surefire.extensions.ForkNodeFactory} that delegates channel creation to Surefire's + * default ({@link SurefireForkNodeFactory}) and decorates the {@link EventHandler} so mvnd can observe per-test + * events. Injected into the surefire/failsafe {@code } config by the daemon; carries the mvnd + * {@code projectId} for attribution. + */ +public class MvndForkNodeFactory extends SurefireForkNodeFactory { + + /** Set by Surefire from the injected {@code ...} configuration. */ + private String projectId; + + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + public String getProjectId() { + return projectId; + } + + @Override + public ForkChannel createForkChannel(ForkNodeArguments arguments) throws IOException { + ForkChannel delegate = super.createForkChannel(arguments); + return new WrappingForkChannel(arguments, delegate, projectId); + } + + /** Wraps a {@link ForkChannel}, decorating the event handler passed to {@link #bindEventHandler}. */ + static final class WrappingForkChannel extends ForkChannel { + private final ForkChannel delegate; + private final String projectId; + private final int forkChannelId; + + WrappingForkChannel(ForkNodeArguments arguments, ForkChannel delegate, String projectId) { + super(arguments); + this.delegate = delegate; + this.projectId = projectId; + this.forkChannelId = arguments.getForkChannelId(); + } + + @Override + public void tryConnectToClient() throws IOException, InterruptedException { + delegate.tryConnectToClient(); + } + + @Override + public String getForkNodeConnectionString() { + return delegate.getForkNodeConnectionString(); + } + + @Override + public int getCountdownCloseablePermits() { + return delegate.getCountdownCloseablePermits(); + } + + @Override + public void bindCommandReader(CommandReader commands, WritableByteChannel stdIn) + throws IOException, InterruptedException { + delegate.bindCommandReader(commands, stdIn); + } + + @Override + public void bindEventHandler( + EventHandler eventHandler, CountdownCloseable countdown, ReadableByteChannel stdOut) + throws IOException, InterruptedException { + delegate.bindEventHandler( + new ProgressEventHandler(projectId, forkChannelId, eventHandler, new TestProgressAccumulator()), + countdown, + stdOut); + } + + @Override + public void disable() { + delegate.disable(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + } + + /** Observes each event, updates the accumulator, pushes through the bridge, then always delegates. */ + static final class ProgressEventHandler implements EventHandler { + private final String projectId; + private final int forkChannelId; + private final EventHandler delegate; + private final TestProgressAccumulator acc; + + ProgressEventHandler( + String projectId, int forkChannelId, EventHandler delegate, TestProgressAccumulator acc) { + this.projectId = projectId; + this.forkChannelId = forkChannelId; + this.delegate = delegate; + this.acc = acc; + } + + @Override + public void handleEvent(Event event) { + try { + observe(event); + } catch (Throwable ignored) { + // Never break the test run because of the progress feature. + } + delegate.handleEvent(event); + } + + private void observe(Event event) { + final TestProgressAccumulator.Type type; + final ReportEntry re; + if (event instanceof TestsetStartingEvent) { + type = TestProgressAccumulator.Type.TESTSET_STARTING; + re = ((TestsetStartingEvent) event).getReportEntry(); + } else if (event instanceof TestStartingEvent) { + type = TestProgressAccumulator.Type.TEST_STARTING; + re = ((TestStartingEvent) event).getReportEntry(); + } else if (event instanceof TestSucceededEvent) { + type = TestProgressAccumulator.Type.TEST_SUCCEEDED; + re = ((TestSucceededEvent) event).getReportEntry(); + } else if (event instanceof TestFailedEvent) { + type = TestProgressAccumulator.Type.TEST_FAILED; + re = ((TestFailedEvent) event).getReportEntry(); + } else if (event instanceof TestErrorEvent) { + type = TestProgressAccumulator.Type.TEST_ERROR; + re = ((TestErrorEvent) event).getReportEntry(); + } else if (event instanceof TestSkippedEvent) { + type = TestProgressAccumulator.Type.TEST_SKIPPED; + re = ((TestSkippedEvent) event).getReportEntry(); + } else if (event instanceof TestsetCompletedEvent) { + type = TestProgressAccumulator.Type.TESTSET_COMPLETED; + re = ((TestsetCompletedEvent) event).getReportEntry(); + } else { + return; // not a test lifecycle event + } + + final String failureMessage = (type == TestProgressAccumulator.Type.TEST_FAILED + || type == TestProgressAccumulator.Type.TEST_ERROR) + ? extractFailureMessage(re) + : null; + acc.record(type, re.getSourceName(), re.getName(), re.getRunMode(), re.getTestRunId(), failureMessage); + + MvndTestProgress listener = MvndTestProgress.getListener(); + if (listener != null) { + listener.update( + projectId, + forkChannelId, + acc.getTestClass(), + acc.getTestMethod(), + acc.getCompleted(), + acc.getFailures(), + acc.getErrors(), + acc.getSkipped(), + acc.getRetrying(), + acc.getFlaky(), + acc.getFlakyTests(), + acc.getFailedTests(), + acc.getErroredTests()); + } + } + + /** Best-effort compact failure message; never throws (caller already guards, but keep it defensive). */ + private static String extractFailureMessage(ReportEntry re) { + StackTraceWriter stw = re.getStackTraceWriter(); + if (stw == null) { + return null; + } + String smart = stw.smartTrimmedStackTrace(); + if (smart != null && !smart.isEmpty()) { + return smart; + } + SafeThrowable throwable = stw.getThrowable(); + return throwable != null ? throwable.getMessage() : null; + } + } +} diff --git a/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/MvndSurefireProgressLocator.java b/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/MvndSurefireProgressLocator.java new file mode 100644 index 000000000..925ac86e3 --- /dev/null +++ b/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/MvndSurefireProgressLocator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.forknode; + +/** + * Zero-dependency marker used by the daemon to locate this module's jar on disk + * (via {@code getProtectionDomain().getCodeSource().getLocation()}) so it can be added to the Surefire plugin realm. + * Deliberately imports nothing from Surefire so it links in the daemon realm. + */ +public final class MvndSurefireProgressLocator { + private MvndSurefireProgressLocator() {} +} diff --git a/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/TestProgressAccumulator.java b/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/TestProgressAccumulator.java new file mode 100644 index 000000000..9a9466825 --- /dev/null +++ b/surefire-progress/src/main/java/org/mvndaemon/mvnd/forknode/TestProgressAccumulator.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.forknode; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.maven.surefire.api.report.RunMode; + +/** + * Accumulates per-fork test counts and the currently executing class/method. Not thread-safe: Surefire delivers + * fork-reader events on a single thread per fork channel. + */ +public class TestProgressAccumulator { + + public enum Type { + TESTSET_STARTING, + TEST_STARTING, + TEST_SUCCEEDED, + TEST_FAILED, + TEST_ERROR, + TEST_SKIPPED, + TESTSET_COMPLETED + } + + private int completed; + private int failures; + private int errors; + private int skipped; + private int retrying; + private int flaky; + private String testClass; + private String testMethod; + private final Map tests = new LinkedHashMap<>(); + + public void record(Type type, String testClass, String testMethod) { + record(type, testClass, testMethod, RunMode.NORMAL_RUN, null, null); + } + + public void record(Type type, String testClass, String testMethod, RunMode runMode, Long testRunId) { + record(type, testClass, testMethod, runMode, testRunId, null); + } + + public void record( + Type type, String testClass, String testMethod, RunMode runMode, Long testRunId, String failureMessage) { + switch (type) { + case TESTSET_STARTING: + this.testClass = testClass; + this.testMethod = null; + break; + case TEST_STARTING: + this.testClass = testClass; + this.testMethod = testMethod; + state(testClass, testMethod, testRunId).starting(runMode); + break; + case TEST_SUCCEEDED: + state(testClass, testMethod, testRunId).succeeded(); + break; + case TEST_FAILED: + state(testClass, testMethod, testRunId).failed(runMode, sanitize(failureMessage)); + break; + case TEST_ERROR: + state(testClass, testMethod, testRunId).errored(runMode, sanitize(failureMessage)); + break; + case TEST_SKIPPED: + state(testClass, testMethod, testRunId).skipped(); + break; + case TESTSET_COMPLETED: + finalizeRetrying(); + break; + } + recompute(); + } + + public int getCompleted() { + return completed; + } + + public int getFailures() { + return failures; + } + + public int getErrors() { + return errors; + } + + public int getSkipped() { + return skipped; + } + + public int getRetrying() { + return retrying; + } + + public int getFlaky() { + return flaky; + } + + public String getTestClass() { + return testClass; + } + + public String getTestMethod() { + return testMethod; + } + + public List getFlakyTests() { + List result = new ArrayList<>(); + for (TestState state : tests.values()) { + if (state.isFlaky()) { + result.add(state.flakyDetail()); + } + } + return result; + } + + public List getFailedTests() { + List result = new ArrayList<>(); + for (TestState state : tests.values()) { + if (state.isFailed()) { + result.add(state.failureLine()); + } + } + return result; + } + + public List getErroredTests() { + List result = new ArrayList<>(); + for (TestState state : tests.values()) { + if (state.isErrored()) { + result.add(state.failureLine()); + } + } + return result; + } + + private static String sanitize(String message) { + if (message == null) { + return null; + } + String flattened = message.replaceAll("\\s+", " ").trim(); + return flattened.isEmpty() ? null : flattened; + } + + private TestState state(String testClass, String testMethod, Long testRunId) { + String key = testRunId != null ? String.valueOf(testRunId) : testClass + "#" + testMethod; + TestState state = tests.get(key); + if (state == null) { + state = new TestState(testClass, testMethod); + tests.put(key, state); + } else { + state.updateName(testClass, testMethod); + } + return state; + } + + private void finalizeRetrying() { + for (TestState state : tests.values()) { + state.finalizeRetrying(); + } + } + + private void recompute() { + completed = 0; + failures = 0; + errors = 0; + skipped = 0; + retrying = 0; + flaky = 0; + + for (TestState state : tests.values()) { + if (state.skipped) { + completed++; + skipped++; + } else if (state.success) { + completed++; + if (state.failure || state.error) { + flaky++; + } + } else if (state.retrying) { + retrying++; + } else if (state.error) { + completed++; + errors++; + } else if (state.failure) { + completed++; + failures++; + } + } + } + + private static final class TestState { + private String testClass; + private String testMethod; + private boolean failure; + private boolean error; + private boolean success; + private boolean skipped; + private boolean retrying; + private String message; + private final List runs = new ArrayList<>(); + + private TestState(String testClass, String testMethod) { + this.testClass = testClass; + this.testMethod = testMethod; + } + + private void updateName(String testClass, String testMethod) { + if (testClass != null) { + this.testClass = testClass; + } + if (testMethod != null) { + this.testMethod = testMethod; + } + } + + private void starting(RunMode runMode) { + if (runMode == RunMode.RERUN_TEST_AFTER_FAILURE) { + retrying = true; + } + } + + private void succeeded() { + success = true; + retrying = false; + runs.add(Run.pass()); + } + + private void failed(RunMode runMode, String failureMessage) { + failure = true; + if (message == null) { + message = failureMessage; + } + runs.add(Run.fail(failureMessage)); + if (runMode == RunMode.RERUN_TEST_AFTER_FAILURE) { + retrying = true; + } + } + + private void errored(RunMode runMode, String failureMessage) { + error = true; + if (message == null) { + message = failureMessage; + } + runs.add(Run.error(failureMessage)); + if (runMode == RunMode.RERUN_TEST_AFTER_FAILURE) { + retrying = true; + } + } + + private void skipped() { + skipped = true; + } + + private void finalizeRetrying() { + retrying = false; + } + + private boolean isFlaky() { + return success && (failure || error); + } + + private boolean isErrored() { + return !skipped && !success && !retrying && error; + } + + private boolean isFailed() { + return !skipped && !success && !retrying && !error && failure; + } + + private String displayName() { + if (testClass == null) { + return testMethod != null ? testMethod : "(unknown)"; + } + String simpleClass = testClass.substring(testClass.lastIndexOf('.') + 1); + return testMethod != null ? simpleClass + "#" + testMethod : simpleClass; + } + + private String failureLine() { + return message != null ? displayName() + ": " + message : displayName(); + } + + private String flakyDetail() { + StringBuilder sb = new StringBuilder(displayName()); + for (int i = 0; i < runs.size(); i++) { + sb.append('\n') + .append(" Run ") + .append(i + 1) + .append(": ") + .append(runs.get(i).describe()); + } + return sb.toString(); + } + + private static final class Run { + private final Outcome outcome; + private final String message; + + private Run(Outcome outcome, String message) { + this.outcome = outcome; + this.message = message; + } + + private static Run pass() { + return new Run(Outcome.PASS, null); + } + + private static Run fail(String message) { + return new Run(Outcome.FAIL, message); + } + + private static Run error(String message) { + return new Run(Outcome.ERROR, message); + } + + private String describe() { + if (outcome == Outcome.PASS) { + return "PASS"; + } + if (message != null) { + return message; + } + return outcome == Outcome.ERROR ? "ERROR" : "FAIL"; + } + + private enum Outcome { + PASS, + FAIL, + ERROR + } + } + } +} diff --git a/surefire-progress/src/test/java/org/mvndaemon/mvnd/forknode/MvndForkNodeFactoryTest.java b/surefire-progress/src/test/java/org/mvndaemon/mvnd/forknode/MvndForkNodeFactoryTest.java new file mode 100644 index 000000000..9a930f144 --- /dev/null +++ b/surefire-progress/src/test/java/org/mvndaemon/mvnd/forknode/MvndForkNodeFactoryTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.forknode; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.maven.surefire.api.event.ControlByeEvent; +import org.apache.maven.surefire.api.event.Event; +import org.apache.maven.surefire.extensions.EventHandler; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mvndaemon.mvnd.testprogress.MvndTestProgress; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class MvndForkNodeFactoryTest { + + @AfterEach + void clearListener() { + MvndTestProgress.setListener(null); + } + + @Test + void alwaysDelegatesEvenWhenListenerThrows() { + List delegated = new ArrayList<>(); + EventHandler real = delegated::add; + + // A listener that always blows up must not prevent delegation to the real handler. + MvndTestProgress.setListener((p, fork, c, m, comp, f, e, s, r, fl, flakyTests, failedTests, erroredTests) -> { + throw new RuntimeException("boom"); + }); + + EventHandler wrapper = + new MvndForkNodeFactory.ProgressEventHandler("proj", 7, real, new TestProgressAccumulator()); + + wrapper.handleEvent(new ControlByeEvent()); + + assertEquals(1, delegated.size(), "the real handler must always be called"); + } + + @Test + void delegatesWhenNoListenerRegistered() { + List delegated = new ArrayList<>(); + EventHandler real = delegated::add; + + EventHandler wrapper = + new MvndForkNodeFactory.ProgressEventHandler("proj", 7, real, new TestProgressAccumulator()); + + wrapper.handleEvent(new ControlByeEvent()); + + assertEquals(1, delegated.size(), "non-test events still pass through"); + } +} diff --git a/surefire-progress/src/test/java/org/mvndaemon/mvnd/forknode/TestProgressAccumulatorTest.java b/surefire-progress/src/test/java/org/mvndaemon/mvnd/forknode/TestProgressAccumulatorTest.java new file mode 100644 index 000000000..a3bc503b0 --- /dev/null +++ b/surefire-progress/src/test/java/org/mvndaemon/mvnd/forknode/TestProgressAccumulatorTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.mvndaemon.mvnd.forknode; + +import org.apache.maven.surefire.api.report.RunMode; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mvndaemon.mvnd.forknode.TestProgressAccumulator.Type.TESTSET_COMPLETED; +import static org.mvndaemon.mvnd.forknode.TestProgressAccumulator.Type.TESTSET_STARTING; +import static org.mvndaemon.mvnd.forknode.TestProgressAccumulator.Type.TEST_ERROR; +import static org.mvndaemon.mvnd.forknode.TestProgressAccumulator.Type.TEST_FAILED; +import static org.mvndaemon.mvnd.forknode.TestProgressAccumulator.Type.TEST_SKIPPED; +import static org.mvndaemon.mvnd.forknode.TestProgressAccumulator.Type.TEST_STARTING; +import static org.mvndaemon.mvnd.forknode.TestProgressAccumulator.Type.TEST_SUCCEEDED; + +class TestProgressAccumulatorTest { + + @Test + void countsPassingTests() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TESTSET_STARTING, "MyServiceTest", null); + acc.record(TEST_STARTING, "MyServiceTest", "shouldWork"); + acc.record(TEST_SUCCEEDED, "MyServiceTest", "shouldWork"); + acc.record(TEST_STARTING, "MyServiceTest", "alsoWorks"); + acc.record(TEST_SUCCEEDED, "MyServiceTest", "alsoWorks"); + + assertEquals(2, acc.getCompleted()); + assertEquals(0, acc.getFailures()); + assertEquals(0, acc.getErrors()); + assertEquals(0, acc.getSkipped()); + assertEquals("MyServiceTest", acc.getTestClass()); + assertEquals("alsoWorks", acc.getTestMethod()); + } + + @Test + void countsFailuresErrorsAndSkips() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TEST_STARTING, "T", "a"); + acc.record(TEST_FAILED, "T", "a"); + acc.record(TEST_STARTING, "T", "b"); + acc.record(TEST_ERROR, "T", "b"); + acc.record(TEST_STARTING, "T", "c"); + acc.record(TEST_SKIPPED, "T", "c"); + + assertEquals(3, acc.getCompleted()); + assertEquals(1, acc.getFailures()); + assertEquals(1, acc.getErrors()); + assertEquals(1, acc.getSkipped()); + } + + @Test + void failedAndErroredTestsExposeNameAndMessage() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TEST_STARTING, "org.example.CalcTest", "adds", RunMode.NORMAL_RUN, null); + acc.record(TEST_FAILED, "org.example.CalcTest", "adds", RunMode.NORMAL_RUN, null, "expected: <5> but was: <4>"); + acc.record(TEST_STARTING, "org.example.CalcTest", "divides", RunMode.NORMAL_RUN, null); + acc.record(TEST_ERROR, "org.example.CalcTest", "divides", RunMode.NORMAL_RUN, null, "/ by zero"); + + assertEquals(1, acc.getFailures()); + assertEquals(1, acc.getErrors()); + assertEquals(java.util.List.of("CalcTest#adds: expected: <5> but was: <4>"), acc.getFailedTests()); + assertEquals(java.util.List.of("CalcTest#divides: / by zero"), acc.getErroredTests()); + } + + @Test + void firstFailureMessageWinsAndNewlinesAreFlattened() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TEST_STARTING, "T", "a", RunMode.NORMAL_RUN, null); + acc.record(TEST_FAILED, "T", "a", RunMode.NORMAL_RUN, null, "line one\n line two"); + + assertEquals(java.util.List.of("T#a: line one line two"), acc.getFailedTests()); + } + + @Test + void flakyTestIsNeitherFailedNorErrored() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TEST_STARTING, "T", "a", RunMode.NORMAL_RUN, 1L); + acc.record(TEST_FAILED, "T", "a", RunMode.NORMAL_RUN, 1L, "boom"); + acc.record(TEST_STARTING, "T", "a", RunMode.RERUN_TEST_AFTER_FAILURE, 1L); + acc.record(TEST_SUCCEEDED, "T", "a", RunMode.RERUN_TEST_AFTER_FAILURE, 1L); + + assertTrue(acc.getFailedTests().isEmpty(), "a recovered test must not be listed as failed"); + assertTrue(acc.getErroredTests().isEmpty(), "a recovered test must not be listed as errored"); + assertEquals(java.util.List.of("T#a\n Run 1: boom\n Run 2: PASS"), acc.getFlakyTests()); + } + + @Test + void testsetStartingSetsClassWithNullMethod() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TESTSET_STARTING, "OtherTest", null); + assertEquals("OtherTest", acc.getTestClass()); + assertNull(acc.getTestMethod()); + } + + @Test + void retriesCanRecoverAsFlakyTests() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TEST_STARTING, "MyServiceTest", "shouldWork", RunMode.NORMAL_RUN, 1L); + acc.record(TEST_FAILED, "MyServiceTest", "shouldWork", RunMode.NORMAL_RUN, 1L); + assertEquals(1, acc.getFailures()); + assertEquals(0, acc.getRetrying()); + + acc.record(TEST_STARTING, "MyServiceTest", "shouldWork", RunMode.RERUN_TEST_AFTER_FAILURE, 1L); + assertEquals(0, acc.getFailures()); + assertEquals(1, acc.getRetrying()); + + acc.record(TEST_SUCCEEDED, "MyServiceTest", "shouldWork", RunMode.RERUN_TEST_AFTER_FAILURE, 1L); + + assertEquals(1, acc.getCompleted()); + assertEquals(0, acc.getFailures()); + assertEquals(0, acc.getRetrying()); + assertEquals(1, acc.getFlaky()); + assertEquals(java.util.List.of("MyServiceTest#shouldWork\n Run 1: FAIL\n Run 2: PASS"), acc.getFlakyTests()); + } + + @Test + void flakyTestDetailListsEachRunWithMessage() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TEST_STARTING, "org.example.FlakyTest", "retries", RunMode.NORMAL_RUN, 7L); + acc.record(TEST_FAILED, "org.example.FlakyTest", "retries", RunMode.NORMAL_RUN, 7L, "expected <5> but was <0>"); + acc.record(TEST_STARTING, "org.example.FlakyTest", "retries", RunMode.RERUN_TEST_AFTER_FAILURE, 7L); + acc.record( + TEST_ERROR, + "org.example.FlakyTest", + "retries", + RunMode.RERUN_TEST_AFTER_FAILURE, + 7L, + "NullPointerException"); + acc.record(TEST_STARTING, "org.example.FlakyTest", "retries", RunMode.RERUN_TEST_AFTER_FAILURE, 7L); + acc.record(TEST_SUCCEEDED, "org.example.FlakyTest", "retries", RunMode.RERUN_TEST_AFTER_FAILURE, 7L); + + assertEquals( + java.util.List.of("FlakyTest#retries\n" + + " Run 1: expected <5> but was <0>\n" + + " Run 2: NullPointerException\n" + + " Run 3: PASS"), + acc.getFlakyTests()); + } + + @Test + void unrecoveredRetryEndsAsFailure() { + TestProgressAccumulator acc = new TestProgressAccumulator(); + acc.record(TEST_STARTING, "MyServiceTest", "shouldWork", RunMode.NORMAL_RUN, 1L); + acc.record(TEST_FAILED, "MyServiceTest", "shouldWork", RunMode.NORMAL_RUN, 1L); + acc.record(TEST_STARTING, "MyServiceTest", "shouldWork", RunMode.RERUN_TEST_AFTER_FAILURE, 1L); + assertEquals(1, acc.getRetrying()); + + acc.record(TESTSET_COMPLETED, "MyServiceTest", null, RunMode.RERUN_TEST_AFTER_FAILURE, 1L); + + assertEquals(1, acc.getCompleted()); + assertEquals(1, acc.getFailures()); + assertEquals(0, acc.getRetrying()); + assertEquals(0, acc.getFlaky()); + } +}