Skip to content

Surface individual rerun attempts as separate test runs #1142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
trunk-io merged 3 commits into from
Jul 14, 2026
Merged

Surface individual rerun attempts as separate test runs #1142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
df21a3f
fix(context): emit a run per Playwright rerun/flaky attempt
claude Jul 13, 2026
6f44690
test(cli): derive expected run count from parsed reruns
claude Jul 13, 2026
7377709
refactor(context): iterate reruns by reference; assert full run vec i…
claude Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions cli/tests/upload.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,33 @@ use test_utils::{
};
use trunk_analytics_cli::upload_command::{DRY_RUN_OUTPUT_DIR, get_bundle_upload_id_message};

// The mock JUnit generator adds a random number of reruns per test, so the number of
// `TestCaseRun`s equals the test cases plus every parsed rerun/flaky attempt. Mirror
// `into_test_case_runs` to derive the expected count from the bundled JUnit rather than hardcoding
// a seed-dependent number.
fn expected_test_case_run_count(reports: &[quick_junit::Report]) -> usize {
reports
.iter()
.flat_map(|report| report.test_suites.iter())
.flat_map(|test_suite| test_suite.test_cases.iter())
.map(|test_case| {
1 + match &test_case.status {
quick_junit::TestCaseStatus::Success { flaky_runs } => flaky_runs.len(),
quick_junit::TestCaseStatus::NonSuccess { reruns, .. } => reruns.len(),
quick_junit::TestCaseStatus::Skipped { .. } => 0,
}
})
.sum()
}

fn expected_test_case_run_count_from_junit(path: &std::path::Path) -> usize {
let mut junit_parser = JunitParser::new();
junit_parser
.parse(BufReader::new(fs::File::open(path).unwrap()))
.unwrap();
expected_test_case_run_count(&junit_parser.into_reports())
}

// NOTE: must be multi threaded to start a mock server
#[tokio::test(flavor = "multi_thread")]
async fn upload_bundle() {
Expand Down Expand Up @@ -218,7 +245,10 @@ async fn upload_bundle() {
assert_eq!(report.test_results.len(), 1);
let report = report.test_results.first().unwrap();
assert_eq!(report.test_build_information, None);
assert_eq!(report.test_case_runs.len(), 500);
assert_eq!(
report.test_case_runs.len(),
expected_test_case_run_count_from_junit(&tar_extract_directory.join(&bundled_file.path))
);
let test_case_run = &report.test_case_runs[0];
assert!(test_case_run.id.is_empty());
assert!(!test_case_run.name.is_empty());
Expand Down Expand Up @@ -328,6 +358,7 @@ async fn upload_bundle_using_bep() {

assert!(!bundle_meta.base_props.file_sets.is_empty());
assert_eq!(bundle_meta.base_props.test_collection, None);
let mut expected_runs = 0usize;
bundle_meta
.base_props
.file_sets
Expand All @@ -340,7 +371,9 @@ async fn upload_bundle_using_bep() {
assert!(junit_parser.parse(BufReader::new(junit_file)).is_ok());
assert!(junit_parser.issues().is_empty());
});
let report = junit_parser.into_reports().pop().unwrap();
let mut reports = junit_parser.into_reports();
expected_runs += expected_test_case_run_count(&reports);
let report = reports.pop().unwrap();
let test_runner_report = file_set.test_runner_report.clone().unwrap();
assert_eq!(
test_runner_report.resolved_status,
Expand Down Expand Up @@ -376,7 +409,7 @@ async fn upload_bundle_using_bep() {

assert_eq!(report.test_results.len(), 1);
let report = report.test_results.first().unwrap();
assert_eq!(report.test_case_runs.len(), 500);
assert_eq!(report.test_case_runs.len(), expected_runs);
assert!(report.test_build_information.is_some());
let test_build_information = assert_matches!(
report.test_build_information.as_ref(),
Expand Down
253 changes: 247 additions & 6 deletions context/src/junit/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ impl JunitParser {
});
}
}
test_case_run.status = match test_case.status {
test_case_run.status = match &test_case.status {
TestCaseStatus::Success { .. } => TestCaseRunStatus::Success.into(),
TestCaseStatus::Skipped {
message,
Expand All @@ -316,8 +316,14 @@ impl JunitParser {
|| system_err.is_some()
{
test_case_run.test_output = Some(TestOutput {
message: message.map(|m| m.to_string()).unwrap_or_default(),
text: description.map(|d| d.to_string()).unwrap_or_default(),
message: message
.as_ref()
.map(|m| m.to_string())
.unwrap_or_default(),
text: description
.as_ref()
.map(|d| d.to_string())
.unwrap_or_default(),
system_out: system_out
.map(|s| s.to_string())
.unwrap_or_default(),
Expand Down Expand Up @@ -351,8 +357,14 @@ impl JunitParser {
|| system_err.is_some()
{
test_case_run.test_output = Some(TestOutput {
message: message.map(|m| m.to_string()).unwrap_or_default(),
text: description.map(|d| d.to_string()).unwrap_or_default(),
message: message
.as_ref()
.map(|m| m.to_string())
.unwrap_or_default(),
text: description
.as_ref()
.map(|d| d.to_string())
.unwrap_or_default(),
system_out: system_out
.map(|s| s.to_string())
.unwrap_or_default(),
Expand Down Expand Up @@ -449,6 +461,16 @@ impl JunitParser {
));
}

let reruns = match &test_case.status {
TestCaseStatus::Success { flaky_runs } => Some(flaky_runs.iter()),
TestCaseStatus::NonSuccess { reruns, .. } => Some(reruns.iter()),
TestCaseStatus::Skipped { .. } => None,
};
if let Some(reruns) = reruns {
for rerun in reruns {
test_case_runs.push(test_case_run_for_rerun(&test_case_run, rerun));
}
}
test_case_runs.push(test_case_run);
}
}
Expand Down Expand Up @@ -920,6 +942,62 @@ fn resolve_non_success_system_output<T: Clone>(
)
}

/// Builds a `TestCaseRun` for a prior rerun/flaky attempt, inheriting `base`'s identity (test id,
/// file, codeowners, quarantine state, ...) but recording the attempt as a failure with its own
/// output and timing. Emitting these alongside the final attempt is what lets pass-on-retry flakes
/// surface — otherwise every attempt collapses into the single final run.
fn test_case_run_for_rerun(base: &TestCaseRun, rerun: &TestRerun) -> TestCaseRun {
let mut run = base.clone();
run.status = TestCaseRunStatus::Failure.into();

let message = rerun.message.as_deref();
let description = rerun.description.as_deref();
let system_out = rerun.system_out.as_deref();
let system_err = rerun.system_err.as_deref();

run.status_output_message = description
.or(message)
.map(str::to_string)
.unwrap_or_default();

run.test_output = if message.is_some()
|| description.is_some()
|| system_out.is_some()
|| system_err.is_some()
{
Some(TestOutput {
message: message.map(str::to_string).unwrap_or_default(),
text: description.map(str::to_string).unwrap_or_default(),
system_out: system_out.map(str::to_string).unwrap_or_default(),
system_err: system_err.map(str::to_string).unwrap_or_default(),
})
} else {
None
};

if let Some(timestamp) = rerun.timestamp {
run.started_at = Some(Timestamp {
seconds: timestamp.timestamp(),
nanos: timestamp.timestamp_subsec_nanos() as i32,
});
}
if let Some(time) = rerun.time {
if run.started_at.is_none() {
run.started_at = Some(Timestamp {
seconds: 0,
nanos: 0,
});
}
run.finished_at = run.started_at.clone().map(|mut v| {
v.seconds += time.as_secs() as i64;
v.nanos += time.subsec_nanos() as i32;
v
});
}

run
}

/// Converts `//pkg/path:target` to `Some("pkg/path")` for codeowners lookup.
/// Returns `None` if the label has no package component (e.g. `//:target`).
fn bazel_label_to_package_path(label: &str) -> Option<String> {
Expand Down Expand Up @@ -1122,7 +1200,9 @@ mod tests {
use std::io::BufReader;

use prost_wkt_types::Timestamp;
use proto::test_context::test_run::{AttemptNumber, LineNumber, TestCaseRunStatus, TestOutput};
use proto::test_context::test_run::{
AttemptNumber, LineNumber, TestCaseRun, TestCaseRunStatus, TestOutput,
};

use crate::{
junit::parser::{IntoTestCaseRunsOptions, JunitParser, bazel_label_to_package_path},
Expand Down Expand Up @@ -1881,4 +1961,165 @@ failures:
let test_output = failing.test_output.as_ref().unwrap();
assert!(test_output.system_out.is_empty());
}

#[test]

@dfrankland dfrankland Jul 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

an aside: this file is too long, we should try to move these tests to the integration tests directory

@EliSchleifer EliSchleifer Jul 14, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed the file has gotten long. I kept the test relocation out of this PR to keep the fix focused, but happy to move these (and the existing into_test_case_runs tests) into the integration tests dir as a follow-up.

Generated by Claude Code

#[allow(deprecated)]
fn test_into_test_case_runs_emits_run_per_flaky_rerun() {
// Playwright reports a pass-on-retry as a passing <testcase> carrying a <flakyError> for
// each failed attempt. Each attempt must surface as its own run so the failed attempt is
// not lost and the test can be classified flaky.
let mut junit_parser = JunitParser::new();
let file_contents = r#"
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="team-settings.pwtest.ts" timestamp="2026-07-08T00:45:39Z" tests="1" failures="0" errors="0">
<testcase name="should show error toast" classname="team-settings.pwtest.ts" time="1.799">
<flakyError message="Test timeout of 5000ms exceeded." type="Error" time="5.595">
<stackTrace><![CDATA[Test timeout of 5000ms exceeded.]]></stackTrace>
<system-out><![CDATA[attempt output]]></system-out>
</flakyError>
</testcase>
</testsuite>
</testsuites>
"#;
junit_parser
.parse(BufReader::new(file_contents.as_bytes()))
.unwrap();

let test_case_runs = junit_parser.into_test_case_runs(IntoTestCaseRunsOptions {
org_slug: "org",
repo: &RepoUrlParts {
host: "host".into(),
owner: "owner".into(),
name: "name".into(),
},
codeowners: None,
quarantined_test_ids: &[],
variant: "",
test_runner_config: None,
});

// The failed attempt is emitted first (as a failure carrying its own output), the final
// passing attempt last; both describe the same test.
let shared = TestCaseRun {
name: "should show error toast".into(),
parent_name: "team-settings.pwtest.ts".into(),
classname: "team-settings.pwtest.ts".into(),
started_at: Some(Timestamp {
seconds: 1783471539,
nanos: 0,
}),
..Default::default()
};
assert_eq!(
test_case_runs,
vec![
TestCaseRun {
status: TestCaseRunStatus::Failure as i32,
status_output_message: "Test timeout of 5000ms exceeded.".into(),
test_output: Some(TestOutput {
message: "Test timeout of 5000ms exceeded.".into(),
text: String::new(),
system_out: "attempt output".into(),
system_err: String::new(),
}),
finished_at: Some(Timestamp {
seconds: 1783471544,
nanos: 595_000_000,
}),
..shared.clone()
},
TestCaseRun {
status: TestCaseRunStatus::Success as i32,
finished_at: Some(Timestamp {
seconds: 1783471540,
nanos: 799_000_000,
}),
..shared
},
]
);
}

#[test]
#[allow(deprecated)]
fn test_into_test_case_runs_emits_run_per_failing_rerun() {
// A test that fails every attempt records the earlier <rerunFailure> attempts as their own
// runs in addition to the final failure.
let mut junit_parser = JunitParser::new();
let file_contents = r#"
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="suite" timestamp="2026-07-08T00:45:39Z" tests="1" failures="1" errors="0">
<testcase name="always_fails" classname="suite" time="1.0">
<failure message="final failure" />
<rerunFailure message="first failure" time="0.5" />
</testcase>
</testsuite>
</testsuites>
"#;
junit_parser
.parse(BufReader::new(file_contents.as_bytes()))
.unwrap();

let test_case_runs = junit_parser.into_test_case_runs(IntoTestCaseRunsOptions {
org_slug: "org",
repo: &RepoUrlParts {
host: "host".into(),
owner: "owner".into(),
name: "name".into(),
},
codeowners: None,
quarantined_test_ids: &[],
variant: "",
test_runner_config: None,
});

// The earlier rerun attempt is emitted first, the final failure last; both are recorded as
// failures.
let shared = TestCaseRun {
name: "always_fails".into(),
parent_name: "suite".into(),
classname: "suite".into(),
status: TestCaseRunStatus::Failure as i32,
started_at: Some(Timestamp {
seconds: 1783471539,
nanos: 0,
}),
..Default::default()
};
assert_eq!(
test_case_runs,
vec![
TestCaseRun {
status_output_message: "first failure".into(),
test_output: Some(TestOutput {
message: "first failure".into(),
text: String::new(),
system_out: String::new(),
system_err: String::new(),
}),
finished_at: Some(Timestamp {
seconds: 1783471539,
nanos: 500_000_000,
}),
..shared.clone()
},
TestCaseRun {
status_output_message: "final failure".into(),
test_output: Some(TestOutput {
message: "final failure".into(),
text: String::new(),
system_out: String::new(),
system_err: String::new(),
}),
finished_at: Some(Timestamp {
seconds: 1783471540,
nanos: 0,
}),
..shared
},
]
);
}
}
Loading