feat(forge): support junit xml test reports (#8852)

* feat(forge): support junit xml test reports

* Update crates/forge/bin/cmd/test/mod.rs

Co-authored-by: DaniPopes <[email protected]>

* Changes after review

* Fix clippy

* Support skipped tests with message

* Set reason msg only is Some

---------

Co-authored-by: DaniPopes <[email protected]>

crates/common/src/shell.rs

@@ -75,8 +75,8 @@ impl Shell {
         Self { output, verbosity }
     }
 
-    /// Returns a new shell that conforms to the specified verbosity arguments, where `json` takes
-    /// higher precedence
+    /// Returns a new shell that conforms to the specified verbosity arguments, where `json`
+    /// or `junit` takes higher precedence.
     pub fn from_args(silent: bool, json: bool) -> Self {
         match (silent, json) {
             (_, true) => Self::json(),

crates/forge/Cargo.toml

@@ -46,6 +46,7 @@ serde.workspace = true
 tracing.workspace = true
 yansi.workspace = true
 humantime-serde = "1.1.1"
+chrono.workspace = true
 
 # bin
 forge-doc.workspace = true
@@ -107,6 +108,7 @@ opener = "0.7"
 
 # soldeer
 soldeer.workspace = true
+quick-junit = "0.5.0"
 
 [target.'cfg(unix)'.dependencies]
 tikv-jemallocator = { workspace = true, optional = true }

crates/forge/bin/cmd/test/mod.rs

@@ -1,5 +1,6 @@
 use super::{install, test::filter::ProjectPathsAwareFilter, watch::WatchArgs};
 use alloy_primitives::U256;
+use chrono::Utc;
 use clap::{Parser, ValueHint};
 use eyre::{Context, OptionExt, Result};
 use forge::{
@@ -40,14 +41,17 @@ use foundry_evm::traces::identifier::TraceIdentifiers;
 use regex::Regex;
 use std::{
     collections::{BTreeMap, BTreeSet},
+    fmt::Write,
     path::PathBuf,
     sync::{mpsc::channel, Arc},
-    time::Instant,
+    time::{Duration, Instant},
 };
 use yansi::Paint;
 
 mod filter;
 mod summary;
+
+use quick_junit::{NonSuccessKind, Report, TestCase, TestCaseStatus, TestSuite};
 use summary::TestSummaryReporter;
 
 pub use filter::FilterArgs;
@@ -115,6 +119,10 @@ pub struct TestArgs {
     #[arg(long, help_heading = "Display options")]
     json: bool,
 
+    /// Output test results as JUnit XML report.
+    #[arg(long, conflicts_with = "json", help_heading = "Display options")]
+    junit: bool,
+
     /// Stop running tests after the first failure.
     #[arg(long)]
     pub fail_fast: bool,
@@ -181,7 +189,7 @@ impl TestArgs {
 
     pub async fn run(self) -> Result<TestOutcome> {
         trace!(target: "forge::test", "executing test command");
-        shell::set_shell(shell::Shell::from_args(self.opts.silent, self.json))?;
+        shell::set_shell(shell::Shell::from_args(self.opts.silent, self.json || self.junit))?;
         self.execute_tests().await
     }
 
@@ -300,7 +308,7 @@ impl TestArgs {
         let sources_to_compile = self.get_sources_to_compile(&config, &filter)?;
 
         let compiler = ProjectCompiler::new()
-            .quiet_if(self.json || self.opts.silent)
+            .quiet_if(self.json || self.junit || self.opts.silent)
             .files(sources_to_compile);
 
         let output = compiler.compile(&project)?;
@@ -494,6 +502,12 @@ impl TestArgs {
             return Ok(TestOutcome::new(results, self.allow_failure));
         }
 
+        if self.junit {
+            let results = runner.test_collect(filter);
+            println!("{}", junit_xml_report(&results, verbosity).to_string()?);
+            return Ok(TestOutcome::new(results, self.allow_failure));
+        }
+
         let remote_chain_id = runner.evm_opts.get_remote_chain_id().await;
         let known_contracts = runner.known_contracts.clone();
 
@@ -814,6 +828,49 @@ fn persist_run_failures(config: &Config, outcome: &TestOutcome) {
     }
 }
 
+/// Generate test report in JUnit XML report format.
+fn junit_xml_report(results: &BTreeMap<String, SuiteResult>, verbosity: u8) -> Report {
+    let mut total_duration = Duration::default();
+    let mut junit_report = Report::new("Test run");
+    junit_report.set_timestamp(Utc::now());
+    for (suite_name, suite_result) in results {
+        let mut test_suite = TestSuite::new(suite_name);
+        total_duration += suite_result.duration;
+        test_suite.set_time(suite_result.duration);
+        test_suite.set_system_out(suite_result.summary());
+        for (test_name, test_result) in &suite_result.test_results {
+            let mut test_status = match test_result.status {
+                TestStatus::Success => TestCaseStatus::success(),
+                TestStatus::Failure => TestCaseStatus::non_success(NonSuccessKind::Failure),
+                TestStatus::Skipped => TestCaseStatus::skipped(),
+            };
+            if let Some(reason) = &test_result.reason {
+                test_status.set_message(reason);
+            }
+
+            let mut test_case = TestCase::new(test_name, test_status);
+            test_case.set_time(test_result.duration);
+
+            let mut sys_out = String::new();
+            let result_report = test_result.kind.report();
+            write!(sys_out, "{test_result} {test_name} {result_report}").unwrap();
+            if verbosity >= 2 && !test_result.logs.is_empty() {
+                write!(sys_out, "\\nLogs:\\n").unwrap();
+                let console_logs = decode_console_logs(&test_result.logs);
+                for log in console_logs {
+                    write!(sys_out, "  {log}\\n").unwrap();
+                }
+            }
+
+            test_case.set_system_out(sys_out);
+            test_suite.add_test_case(test_case);
+        }
+        junit_report.add_test_suite(test_suite);
+    }
+    junit_report.set_time(total_duration);
+    junit_report
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/forge/tests/cli/test_cmd.rs

@@ -1977,5 +1977,118 @@ Suite result: ok. 0 passed; 0 failed; 6 skipped; [ELAPSED]
 
 Ran 1 test suite [ELAPSED]: 0 tests passed, 0 failed, 6 skipped (6 total tests)
 
+"#]]);
+});
+
+forgetest_init!(should_generate_junit_xml_report, |prj, cmd| {
+    prj.wipe_contracts();
+    prj.insert_ds_test();
+    prj.insert_vm();
+    prj.clear();
+
+    prj.add_source(
+        "JunitReportTest.t.sol",
+        r#"
+        import {Vm} from "./Vm.sol";
+        import {DSTest} from "./test.sol";
+
+        contract AJunitReportTest is DSTest {
+            function test_junit_assert_fail() public {
+                assert(1 > 2);
+            }
+
+            function test_junit_revert_fail() public {
+                require(1 > 2, "Revert");
+            }
+        }
+
+        contract BJunitReportTest is DSTest {
+            Vm constant vm = Vm(HEVM_ADDRESS);
+            function test_junit_pass() public {
+                require(1 < 2, "Revert");
+            }
+
+            function test_junit_skip() public {
+                vm.skip(true);
+            }
+
+            function test_junit_skip_with_message() public {
+                vm.skip(true, "skipped test");
+            }
+
+            function test_junit_pass_fuzz(uint256 a) public {
+            }
+        }
+   "#,
+    )
+    .unwrap();
+
+    cmd.args(["test", "--junit"]).assert_failure().stdout_eq(str![[r#"
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="Test run" tests="6" failures="2" errors="0" timestamp="[..]" time="[..]">
+    <testsuite name="src/JunitReportTest.t.sol:AJunitReportTest" tests="2" disabled="0" errors="0" failures="2" time="[..]">
+        <testcase name="test_junit_assert_fail()" time="[..]">
+            <failure message="panic: assertion failed (0x01)"/>
+            <system-out>[FAIL: panic: assertion failed (0x01)] test_junit_assert_fail() ([GAS])</system-out>
+        </testcase>
+        <testcase name="test_junit_revert_fail()" time="[..]">
+            <failure message="revert: Revert"/>
+            <system-out>[FAIL: revert: Revert] test_junit_revert_fail() ([GAS])</system-out>
+        </testcase>
+        <system-out>Suite result: FAILED. 0 passed; 2 failed; 0 skipped; [ELAPSED]</system-out>
+    </testsuite>
+    <testsuite name="src/JunitReportTest.t.sol:BJunitReportTest" tests="4" disabled="2" errors="0" failures="0" time="[..]">
+        <testcase name="test_junit_pass()" time="[..]">
+            <system-out>[PASS] test_junit_pass() ([GAS])</system-out>
+        </testcase>
+        <testcase name="test_junit_pass_fuzz(uint256)" time="[..]">
+            <system-out>[PASS] test_junit_pass_fuzz(uint256) (runs: 256, [AVG_GAS])</system-out>
+        </testcase>
+        <testcase name="test_junit_skip()" time="[..]">
+            <skipped/>
+            <system-out>[SKIP] test_junit_skip() ([GAS])</system-out>
+        </testcase>
+        <testcase name="test_junit_skip_with_message()" time="[..]">
+            <skipped message="skipped test"/>
+            <system-out>[SKIP: skipped test] test_junit_skip_with_message() ([GAS])</system-out>
+        </testcase>
+        <system-out>Suite result: ok. 2 passed; 0 failed; 2 skipped; [ELAPSED]</system-out>
+    </testsuite>
+</testsuites>
+
+
+"#]]);
+});
+
+forgetest_init!(should_generate_junit_xml_report_with_logs, |prj, cmd| {
+    prj.wipe_contracts();
+    prj.add_source(
+        "JunitReportTest.t.sol",
+        r#"
+import "forge-std/Test.sol";
+contract JunitReportTest is Test {
+    function test_junit_with_logs() public {
+        console.log("Step1");
+        console.log("Step2");
+        console.log("Step3");
+        assert(2 > 1);
+    }
+}
+   "#,
+    )
+    .unwrap();
+
+    cmd.args(["test", "--junit", "-vvvv"]).assert_success().stdout_eq(str![[r#"
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="Test run" tests="1" failures="0" errors="0" timestamp="[..]" time="[..]">
+    <testsuite name="src/JunitReportTest.t.sol:JunitReportTest" tests="1" disabled="0" errors="0" failures="0" time="[..]">
+        <testcase name="test_junit_with_logs()" time="[..]">
+            <system-out>[PASS] test_junit_with_logs() ([GAS])/nLogs:/n  Step1/n  Step2/n  Step3/n</system-out>
+        </testcase>
+        <system-out>Suite result: ok. 1 passed; 0 failed; 0 skipped; [ELAPSED]</system-out>
+    </testsuite>
+</testsuites>
+
+
 "#]]);
 });