Improve provisioning failure message

mkdocs/docs/guides/troubleshooting.md

@@ -148,16 +148,24 @@ Alternatively, you can configure your own cloud accounts
 on the [project settings page](../concepts/projects.md#backends)
 or use [SSH fleets](../concepts/fleets.md#ssh-fleets).
 
-### Provisioning fails
+### Provisioning fails { #provisioning-fails }
+[//]: # (NOTE: This section is referenced in the CLI. Do not change its URL.)
 
 In certain cases, running `dstack apply` may show instance offers,
 but then produce the following output:
 
 ```shell
 wet-mangust-1 provisioning completed (failed)
-All provisioning attempts failed. This is likely due to cloud providers not having enough capacity. Check CLI and server logs for more details.
+No capacity
+Failed to provision in fleet 'aws-main': tried 5 of 12 offers (attempt limit reached), all failed.
+Errors: g5.xlarge in aws/us-east-1: InsufficientInstanceCapacity; g5.xlarge in aws/eu-west-1: RequestLimitExceeded
 ```
 
+`dstack` only tries offers from the fleet it selected for the run, so the message names that
+fleet, how many of its offers were tried, and what each attempt returned. Repeated errors are
+reported once; every attempt is logged by the server, so check the [server logs](#server-logs)
+for the full list.
+
 #### Cause 1: Insufficient service quotas
 
 If some runs fail to provision, it may be due to an insufficient service quota. For cloud providers like AWS, GCP,

src/dstack/_internal/cli/services/configurators/run.py

@@ -59,7 +59,14 @@
 from dstack._internal.core.models.repos import RepoHeadWithCreds
 from dstack._internal.core.models.repos.base import Repo
 from dstack._internal.core.models.repos.remote import RemoteRepo, RemoteRepoCreds
-from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunPlan, RunSpec, RunStatus
+from dstack._internal.core.models.runs import (
+    JobStatus,
+    JobSubmission,
+    JobTerminationReason,
+    RunPlan,
+    RunSpec,
+    RunStatus,
+)
 from dstack._internal.core.services.diff import diff_models
 from dstack._internal.core.services.repos import get_repo_creds_and_default_branch
 from dstack._internal.core.services.ssh.ports import PortUsedError
@@ -943,9 +950,15 @@ def print_finished_message(run: Run):
         console.print(str)
 
         if termination_reason_message:
-            console.print(f"[error]{termination_reason_message}[/error]")
-
-        if termination_reason:
+            # Backend errors reported in the message contain square brackets and numbers,
+            # which rich would otherwise parse as markup or repaint.
+            console.print(termination_reason_message, style="error", markup=False, highlight=False)
+
+        if (
+            termination_reason
+            # A run that never started has no runner logs to read.
+            and termination_reason != JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY.value
+        ):
             console.print(f"Check [code]dstack logs -d {run.name}[/code] for more details.")
 
 

src/dstack/_internal/cli/utils/run.py

@@ -205,7 +205,7 @@ def _format_run_status(run) -> str:
         RunStatus.FAILED: "indian_red1",
         RunStatus.DONE: "grey",
     }
-    if status_text in ("no offers", "interrupted"):
+    if status_text in ("no capacity", "interrupted"):
         color = "gold1"
     elif status_text == "no fleets":
         color = "indian_red1"
@@ -220,7 +220,7 @@ def _format_run_status(run) -> str:
 def _format_job_submission_status(job_submission: JobSubmission, verbose: bool) -> str:
     status_message = job_submission.status_message
     job_status = job_submission.status
-    if status_message in ("no offers", "interrupted"):
+    if status_message in ("no capacity", "interrupted"):
         color = "gold1"
     elif status_message == "no fleets":
         color = "indian_red1"

src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py

@@ -450,9 +450,39 @@ class _ExistingInstanceProvisioning:
     volume_attachment_result: _VolumeAttachmentResult
 
 
+@dataclass
+class _OfferAttemptError:
+    backend: str
+    region: str
+    instance: str
+    error: str
+
+
+@dataclass
+class _NewCapacityAttempts:
+    """
+    What happened when the offers of the selected fleet were tried.
+    Used to explain why provisioning failed.
+    """
+
+    total: int
+    """Offers matching the run requirements at the time of provisioning."""
+    tried: int
+    """Offers actually attempted. Lower than `total` if offers were skipped
+    or the attempt limit was reached."""
+    skip_reasons: list[str]
+    """Why the offers that could not be attempted at all were skipped."""
+    errors: list[_OfferAttemptError]
+    """Errors of the attempted offers."""
+    limit_reached: bool
+    """Whether the loop stopped at `settings.MAX_OFFERS_TRIED` with offers left."""
+
+
 @dataclass
 class _FailedNewCapacityProvisioning:
     placement_group_cleanup: Optional[_PlacementGroupCleanup]
+    message: Optional[str] = None
+    """Why the job could not be provisioned. `None` if the offers were never tried."""
 
 
 @dataclass
@@ -1449,6 +1479,7 @@ async def _process_new_capacity_provisioning(
         logger.debug("%s: provisioning failed", fmt(context.job_model))
         return _TerminateSubmittedJobResult(
             reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY,
+            message=provision_new_capacity_result.message,
             locked_fleet_id=locked_fleet_id,
             placement_group_cleanup=provision_new_capacity_result.placement_group_cleanup,
         )
@@ -1475,6 +1506,55 @@ async def _process_new_capacity_provisioning(
     )
 
 
+_PROVISIONING_TROUBLESHOOTING_URL = (
+    "https://dstack.ai/docs/guides/troubleshooting/#provisioning-fails"
+)
+_MAX_REPORTED_OFFER_ERRORS = 3
+
+
+def _get_new_capacity_failure_message(
+    fleet_name: str,
+    attempts: _NewCapacityAttempts,
+) -> str:
+    if attempts.total == 0:
+        return (
+            f"No offers matching the run requirements in fleet {fleet_name!r}."
+            f"\nSee {_PROVISIONING_TROUBLESHOOTING_URL}"
+        )
+    if attempts.tried == 0:
+        return (
+            f"None of the {attempts.total} offers in fleet {fleet_name!r} could be tried:"
+            f" {_format_reported_reasons(attempts.skip_reasons)}."
+            f"\nSee {_PROVISIONING_TROUBLESHOOTING_URL}"
+        )
+    limit_reached = " (attempt limit reached)" if attempts.limit_reached else ""
+    message = (
+        f"Failed to provision in fleet {fleet_name!r}:"
+        f" tried {attempts.tried} of {attempts.total} offers{limit_reached}, all failed."
+    )
+    if attempts.errors:
+        message += f"\nErrors: {_format_offer_errors(attempts.errors)}."
+    return f"{message}\nSee {_PROVISIONING_TROUBLESHOOTING_URL}"
+
+
+def _format_offer_errors(errors: list[_OfferAttemptError]) -> str:
+    # Offers commonly fail with the same error, so report every error once.
+    errors_by_message: dict[str, _OfferAttemptError] = {}
+    for error in errors:
+        errors_by_message.setdefault(error.error, error)
+    return _format_reported_reasons(
+        [f"{e.instance} in {e.backend}/{e.region}: {e.error}" for e in errors_by_message.values()]
+    )
+
+
+def _format_reported_reasons(reasons: list[str]) -> str:
+    unique_reasons = list(dict.fromkeys(reasons))
+    reported = unique_reasons[:_MAX_REPORTED_OFFER_ERRORS]
+    if len(unique_reasons) > len(reported):
+        reported.append(f"and {len(unique_reasons) - len(reported)} more")
+    return "; ".join(reported)
+
+
 async def _apply_new_capacity_provisioning(
     session: AsyncSession,
     item: JobSubmittedPipelineItem,
@@ -2233,10 +2313,14 @@ async def _provision_new_capacity(
     )
     offers_iter = iter(offers)
     offers_tried = 0
+    offers_taken = 0
+    skip_reasons: list[str] = []
+    offer_errors: list[_OfferAttemptError] = []
     while offers_tried < settings.MAX_OFFERS_TRIED:
         backend_with_offer = next(offers_iter, None)
         if backend_with_offer is None:
             break
+        offers_taken += 1
         backend, offer = backend_with_offer
         logger.debug(
             "%s: trying %s in %s/%s for $%0.4f per hour",
@@ -2276,6 +2360,7 @@ async def _provision_new_capacity(
                 compute=compute,
             )
             if placement_group_model is None:
+                skip_reasons.append("no compatible placement group")
                 continue
             if placement_group_model.id not in known_placement_group_ids:
                 new_placement_group_models.append(placement_group_model)
@@ -2334,6 +2419,7 @@ async def _provision_new_capacity(
             )
         except SkipOffer as e:
             offers_tried -= 1
+            skip_reasons.append(str(e) or "offer skipped")
             logger.info(
                 "%s: %s launch in %s/%s skipped: %s",
                 fmt(job_model),
@@ -2344,6 +2430,7 @@ async def _provision_new_capacity(
             )
             continue
         except BackendError as e:
+            offer_errors.append(_get_offer_attempt_error(offer=offer, error=e))
             logger.warning(
                 "%s: %s launch in %s/%s failed: %s",
                 fmt(job_model),
@@ -2353,7 +2440,8 @@ async def _provision_new_capacity(
                 repr(e),
             )
             continue
-        except Exception:
+        except Exception as e:
+            offer_errors.append(_get_offer_attempt_error(offer=offer, error=e))
             logger.exception(
                 "%s: got exception when launching %s in %s/%s",
                 fmt(job_model),
@@ -2363,12 +2451,42 @@ async def _provision_new_capacity(
             )
             continue
     return _FailedNewCapacityProvisioning(
+        message=_get_new_capacity_failure_message(
+            fleet_name=fleet_model.name,
+            attempts=_NewCapacityAttempts(
+                total=len(offers),
+                tried=offers_tried,
+                skip_reasons=skip_reasons,
+                errors=offer_errors,
+                # Offers are left only if the attempt limit, not the offer list, ended the loop.
+                limit_reached=offers_taken < len(offers),
+            ),
+        ),
         placement_group_cleanup=_build_placement_group_cleanup(
             fleet_model=fleet_model,
             offers_tried=offers_tried,
             selected_placement_group_id=None,
             new_placement_group_models=new_placement_group_models,
-        )
+        ),
+    )
+
+
+_MAX_OFFER_ERROR_LEN = 200
+
+
+def _get_offer_attempt_error(
+    offer: InstanceOfferWithAvailability,
+    error: Exception,
+) -> _OfferAttemptError:
+    # Backend errors may be multiline and arbitrarily long since they often wrap cloud API errors.
+    message = " ".join(str(error).split()) or type(error).__name__
+    if len(message) > _MAX_OFFER_ERROR_LEN:
+        message = message[:_MAX_OFFER_ERROR_LEN] + "..."
+    return _OfferAttemptError(
+        backend=offer.backend.value,
+        region=offer.region,
+        instance=offer.instance.name,
+        error=message,
     )
 
 

src/dstack/_internal/server/services/jobs/__init__.py

@@ -597,7 +597,7 @@ def _get_job_status_message(job_model: JobModel) -> str:
                 and "No matching fleet found" in job_model.termination_reason_message
             ):
                 return "no fleets"
-            return "no offers"
+            return "no capacity"
         elif job_model.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY:
             return "interrupted"
         else:

src/tests/_internal/cli/utils/test_run.py

@@ -255,7 +255,7 @@ async def test_simple_run(self, session: AsyncSession):
                 JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY,
                 None,
                 None,
-                "no offers",
+                "no capacity",
                 "gold1",
             ),
             (