Skip to content

Improve failing tests lists #953

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
antoyo merged 1 commit into from
Aug 9, 2026
Merged

Improve failing tests lists #953

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 40 additions & 18 deletions build_system/src/test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1002,14 +1002,16 @@ fn contains_ui_error_patterns(file_path: &Path, keep_lto_tests: bool) -> Result<
// * `prepare_files_callback`: A callback function that prepares the files needed for the test. Its used to remove/retain tests giving Error to run various rust test suits.
// * `run_error_pattern_test`: A boolean that determines whether to run only error pattern tests.
// * `test_type`: A string that indicates the type of the test being run.
// * `retained_tests_list_path`: The list of tests that `prepare_files_callback` retained, if any.
// It is checked against the tests remaining after the filtering to report dead lines.
//
fn test_rustc_inner<F>(
env: &Env,
args: &TestArg,
prepare_files_callback: F,
run_error_pattern_test: bool,
test_type: &str,
run_ignored_tests: bool,
retained_tests_list_path: Option<&str>,
) -> Result<(), String>
where
F: Fn(&Path) -> Result<bool, String>,
Expand Down Expand Up @@ -1079,6 +1081,9 @@ where
false,
)?;
}
if let Some(retained_tests_list_path) = retained_tests_list_path {
check_for_dead_listed_tests(&rust_path, retained_tests_list_path)?;
}
let nb_parts = args.nb_parts.unwrap_or(0);
if nb_parts > 0 {
let current_part = args.current_part.unwrap();
Expand Down Expand Up @@ -1137,7 +1142,7 @@ where
env.get_mut("RUSTFLAGS").unwrap().clear();

let test_dir = format!("tests/{test_type}");
let mut command: Vec<&dyn AsRef<OsStr>> = vec![
let command: Vec<&dyn AsRef<OsStr>> = vec![
&"./x.py",
&"test",
&"--run",
Expand All @@ -1152,19 +1157,36 @@ where
&"--bypass-ignore-backends",
];

if run_ignored_tests {
command.push(&"--");
command.push(&"--ignored");
}

run_command_with_output_and_env(&command, Some(&rust_path), Some(&env))?;
Ok(())
}

/// Checks that every test listed in `list_path` survived the filtering done by
/// `contains_ui_error_patterns`.
fn check_for_dead_listed_tests(rust_path: &Path, list_path: &str) -> Result<(), String> {
let listed_tests = std::fs::read_to_string(list_path)
.map_err(|error| format!("Failed to read `{list_path}`: {error:?}"))?;
let dead_tests = listed_tests
.lines()
.map(|line| line.trim())
.filter(|line| !line.is_empty() && !rust_path.join(line).exists())
.collect::<Vec<_>>();
if dead_tests.is_empty() {
return Ok(());
}
Err(format!(
"The following tests listed in `{list_path}` are filtered out before the tests are run, \
so listing them has no effect:\n{}\n\nThis happens when a test contains an error pattern \
(like `//~` or `//@ known-bug`), in which case it should be removed from `{list_path}`, \
or when it uses LTO, in which case it should be moved to `tests/failing-lto-tests.txt`.",
dead_tests.join("\n")
))
}

fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
test_rustc_inner(env, args, |_| Ok(false), false, "run-make", false)?;
test_rustc_inner(env, args, |_| Ok(false), false, "run-make-cargo", false)?;
test_rustc_inner(env, args, |_| Ok(false), false, "ui", false)
test_rustc_inner(env, args, |_| Ok(false), false, "run-make", None)?;
test_rustc_inner(env, args, |_| Ok(false), false, "run-make-cargo", None)?;
test_rustc_inner(env, args, |_| Ok(false), false, "ui", None)
}

fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
Expand All @@ -1174,16 +1196,16 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
retain_files_callback("tests/failing-run-make-tests.txt", "run-make"),
false,
"run-make",
true,
None,
);

let run_make_cargo_result = test_rustc_inner(
env,
args,
retain_files_callback("tests/failing-run-make-tests.txt", "run-make-cargo"),
false,
"run-make",
true,
"run-make-cargo",
None,
);

let ui_result = test_rustc_inner(
Expand All @@ -1192,7 +1214,7 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
retain_files_callback("tests/failing-ui-tests.txt", "ui"),
false,
"ui",
true,
Some("tests/failing-ui-tests.txt"),
);

run_make_result.and(run_make_cargo_result).and(ui_result)
Expand All @@ -1205,23 +1227,23 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
remove_files_callback("tests/failing-ui-tests.txt", "ui"),
false,
"ui",
false,
None,
)?;
test_rustc_inner(
env,
args,
remove_files_callback("tests/failing-run-make-tests.txt", "run-make"),
false,
"run-make",
false,
None,
)?;
test_rustc_inner(
env,
args,
remove_files_callback("tests/failing-run-make-tests.txt", "run-make-cargo"),
false,
"run-make-cargo",
false,
None,
)
}

Expand All @@ -1232,7 +1254,7 @@ fn test_failing_ui_pattern_tests(env: &Env, args: &TestArg) -> Result<(), String
remove_files_callback("tests/failing-ice-tests.txt", "ui"),
true,
"ui",
false,
None,
)
}

Expand Down
2 changes: 2 additions & 0 deletions tests/failing-lto-tests.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
tests/ui/lto/debuginfo-lto-alloc.rs
tests/ui/extern/issue-64655-allow-unwind-when-calling-panic-directly.rs
tests/ui/extern/issue-64655-extern-rust-must-allow-unwind.rs
tests/ui/lto/thin-lto-inlines2.rs
tests/ui/lto/lto-thin-rustc-loads-linker-plugin.rs
33 changes: 0 additions & 33 deletions tests/failing-ui-tests.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,44 +1,11 @@
tests/ui/asm/may_unwind.rs
tests/ui/asm/x86_64/may_unwind.rs
tests/ui/intrinsics/panic-uninitialized-zeroed.rs
tests/ui/consts/missing_span_in_backtrace.rs
tests/ui/simd/issue-17170.rs
tests/ui/simd/issue-39720.rs
tests/ui/process/println-with-broken-pipe.rs
tests/ui/lto/thin-lto-inlines2.rs
tests/ui/panic-runtime/lto-abort.rs
tests/ui/lto/lto-thin-rustc-loads-linker-plugin.rs
tests/ui/async-await/deep-futures-are-freeze.rs
tests/ui/simd/repr_packed.rs
tests/ui/async-await/in-trait/dont-project-to-specializable-projection.rs
tests/ui/consts/const_cmp_type_id.rs
tests/ui/sanitizer/cfi/assoc-ty-lifetime-issue-123053.rs
tests/ui/sanitizer/cfi/async-closures.rs
tests/ui/sanitizer/cfi/closures.rs
tests/ui/sanitizer/cfi/complex-receiver.rs
tests/ui/sanitizer/cfi/coroutine.rs
tests/ui/sanitizer/cfi/drop-in-place.rs
tests/ui/sanitizer/cfi/drop-no-principal.rs
tests/ui/sanitizer/cfi/fn-ptr.rs
tests/ui/sanitizer/cfi/self-ref.rs
tests/ui/sanitizer/cfi/supertraits.rs
tests/ui/sanitizer/cfi/virtual-auto.rs
tests/ui/sanitizer/cfi/sized-associated-ty.rs
tests/ui/sanitizer/cfi/can-reveal-opaques.rs
tests/ui/consts/const-eval/parse_ints.rs
tests/ui/simd/intrinsic/generic-as.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/inline1.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/inline2.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/segfault.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/zero.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/inline1.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/inline2.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/segfault.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/zero.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/inline1.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/inline2.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/segfault.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/zero.rs
tests/ui/simd/simd-bitmask-notpow2.rs
tests/ui/codegen/StackColoring-not-blowup-stack-issue-40883.rs
tests/ui/numbers-arithmetic/u128-as-f32.rs
Expand Down
Loading