Fix nested exception handler reachability in ILLink

src/coreclr/tools/ILTrim.Tests/ILTrimExpectedFailures.txt

@@ -454,6 +454,7 @@ UnreachableBlock.InstanceMethodSubstitutions
 UnreachableBlock.MethodArgumentPropagation
 UnreachableBlock.MethodWithParametersSubstitutions
 UnreachableBlock.MultiStageRemoval
+UnreachableBlock.NestedFinallyInFinallyHandler
 UnreachableBlock.ReplacedJumpTarget
 UnreachableBlock.ReplacedReturns
 UnreachableBlock.ResultInliningNotPossible

src/tools/illink/src/linker/Linker.Steps/UnreachableBlocksOptimizer.cs

@@ -1187,7 +1187,23 @@ BitArray GetReachableInstructionsMap(out List<ExceptionHandler>? unreachableHand
                 var reachable = new BitArray(FoldedInstructions.Count);
 
                 Stack<int>? condBranches = null;
-                bool exceptionHandlersChecked = !Body.HasExceptionHandlers;
+                BitArray? reachableExceptionHandlers = null;
+                (int Start, int End)[]? exceptionHandlerRanges = null;
+                if (Body.HasExceptionHandlers)
+                {
+                    int handlerCount = ExceptionHandlers.Count;
+                    reachableExceptionHandlers = new BitArray(handlerCount);
+                    exceptionHandlerRanges = new (int Start, int End)[handlerCount];
+
+                    // Fixed-point discovery can scan handlers multiple times, but instruction positions do not change here.
+                    Collection<Instruction> instructions = Instructions;
+                    for (int handlerIndex = 0; handlerIndex < handlerCount; handlerIndex++)
+                    {
+                        ExceptionHandler handler = ExceptionHandlers[handlerIndex];
+                        exceptionHandlerRanges[handlerIndex] = (instructions.IndexOf(handler.TryStart), instructions.IndexOf(handler.TryEnd) - 1);
+                    }
+                }
+
                 Instruction target;
                 int i = 0;
                 while (true)
@@ -1245,23 +1261,23 @@ BitArray GetReachableInstructionsMap(out List<ExceptionHandler>? unreachableHand
                         continue;
                     }
 
-                    if (!exceptionHandlersChecked)
+                    if (reachableExceptionHandlers != null)
                     {
-                        exceptionHandlersChecked = true;
+                        Debug.Assert(exceptionHandlerRanges is not null);
 
-                        var instrs = Instructions;
-                        foreach (var handler in ExceptionHandlers)
+                        // Newly reachable handlers can contain protected regions for nested handlers.
+                        for (int handlerIndex = 0; handlerIndex < ExceptionHandlers.Count; handlerIndex++)
                         {
-                            int start = instrs.IndexOf(handler.TryStart);
-                            int end = instrs.IndexOf(handler.TryEnd) - 1;
+                            if (reachableExceptionHandlers[handlerIndex])
+                                continue;
 
-                            if (!HasAnyBitSet(reachable, start, end))
-                            {
-                                unreachableHandlers ??= new List<ExceptionHandler>();
+                            var handler = ExceptionHandlers[handlerIndex];
+                            (int Start, int End) range = exceptionHandlerRanges[handlerIndex];
 
-                                unreachableHandlers.Add(handler);
+                            if (!HasAnyBitSet(reachable, range.Start, range.End))
                                 continue;
-                            }
+
+                            reachableExceptionHandlers[handlerIndex] = true;
 
                             condBranches ??= new Stack<int>();
 
@@ -1289,6 +1305,15 @@ BitArray GetReachableInstructionsMap(out List<ExceptionHandler>? unreachableHand
                             i = condBranches.Pop();
                             continue;
                         }
+
+                        for (int handlerIndex = 0; handlerIndex < ExceptionHandlers.Count; handlerIndex++)
+                        {
+                            if (reachableExceptionHandlers[handlerIndex])
+                                continue;
+
+                            unreachableHandlers ??= new List<ExceptionHandler>();
+                            unreachableHandlers.Add(ExceptionHandlers[handlerIndex]);
+                        }
                     }
 
                     return reachable;

src/tools/illink/test/Mono.Linker.Tests.Cases/UnreachableBlock/NestedFinallyInFinallyHandler.cs

@@ -0,0 +1,57 @@
+using Mono.Linker.Tests.Cases.Expectations.Assertions;
+using Mono.Linker.Tests.Cases.Expectations.Metadata;
+
+namespace Mono.Linker.Tests.Cases.UnreachableBlock
+{
+    [SetupCompileArgument("/optimize+")]
+    [SetupLinkerArgument("--enable-opt", "ipconstprop")]
+    public class NestedFinallyInFinallyHandler
+    {
+        public static void Main()
+        {
+            Test();
+        }
+
+        [Kept]
+        [ExpectBodyModified]
+        static void Test()
+        {
+            try
+            {
+                if (AlwaysFalse)
+                    Unreachable();
+            }
+            finally
+            {
+                try
+                {
+                    Reached();
+                }
+                finally
+                {
+                    try
+                    {
+                        Cleanup();
+                    }
+                    finally
+                    {
+                        NestedCleanup();
+                    }
+                }
+            }
+        }
+
+        static bool AlwaysFalse => false;
+
+        static void Unreachable() { }
+
+        [Kept]
+        static void Reached() { }
+
+        [Kept]
+        static void Cleanup() { }
+
+        [Kept]
+        static void NestedCleanup() { }
+    }
+}