Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 16 additions & 106 deletions src/main/kotlin/com/redhat/devtools/gateway/server/RemoteIDEServer.kt
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import com.google.gson.Gson
import com.intellij.openapi.diagnostic.thisLogger
import com.redhat.devtools.gateway.DevSpacesContext
import com.redhat.devtools.gateway.openshift.DevWorkspacePods
import com.redhat.devtools.gateway.util.isCancellationException
import io.kubernetes.client.openapi.models.V1Container
import io.kubernetes.client.openapi.models.V1Pod
import kotlinx.coroutines.*
Expand Down Expand Up @@ -45,9 +44,6 @@ class RemoteIDEServer(private val devSpacesContext: DevSpacesContext) {
* status exec cannot burn the entire wait (CRW-11119).
*/
const val STATUS_EXEC_TIMEOUT: Long = 15 // seconds

/** Number of consecutive pod-refresh failures before emitting a warning. */
const val REFRESH_FAILURE_WARNING_THRESHOLD: Int = 10
}

init {
Expand Down Expand Up @@ -112,121 +108,35 @@ class RemoteIDEServer(private val devSpacesContext: DevSpacesContext) {
*/
@Throws(IOException::class)
suspend fun waitServerReady(checkCancelled: (() -> Unit)? = null, timeout: Long = readyTimeout): Boolean {
return doWaitServerState(true, timeout, checkCancelled)
return waitForState(true, timeout, checkCancelled)
.also {
if (!it) throw IOException(
"Workspace IDE is not ready after $timeout seconds.",
)
}
}

/**
* Re-resolves the workspace pod and idea-server container.
*
* @return `true` when refreshed successfully, `false` on transient failures (retried).
* Terminal conditions (cancellation, missing idea-server container) are rethrown.
*/
@Throws(CancellationException::class)
private fun refreshPod(refreshFailures: IntArray): Boolean {
return try {
pod = findPod()
container = findContainer()
refreshFailures[0] = 0
true
} catch (e: Exception) {
if (e.isCancellationException()) throw e
if (e is ServerContainerNotFoundException) throw e
refreshFailures[0]++
thisLogger().debug("Failed to refresh workspace pod during IDE state check", e)
if (refreshFailures[0] == REFRESH_FAILURE_WARNING_THRESHOLD) {
thisLogger().warn(
"Pod/container refresh has failed ${refreshFailures[0]} consecutive times; " +
"stale pod references may cause incorrect status checks"
)
}
false
}
}

@Throws(CancellationException::class)
private suspend fun isServerState(
isReadyState: Boolean,
checkCancelled: (() -> Unit)? = null,
refreshPodBeforeCheck: Boolean = false,
refreshFailures: IntArray = intArrayOf(0),
): Boolean {
return try {
// Re-resolve pod while waiting for ready so a recycled pod is not missed.
if (refreshPodBeforeCheck && !refreshPod(refreshFailures)) {
return false
}
getStatus(checkCancelled).isReady == isReadyState
} catch (e: Exception) {
if (e.isCancellationException()) throw e
if (e is ServerContainerNotFoundException) throw e
thisLogger().debug("Failed to check workspace IDE state.", e)
false
}
}

@Throws(IOException::class)
suspend fun waitServerTerminated(timeout: Long = 10L): Boolean {
return doWaitServerState(false, timeout)
return waitForState(false, timeout)
}

/**
* Waits for the server to have or not have projects according to the given parameter.
* Times out the wait if the expected state is not reached within specified timeout.
*
* @param isReadyState True if server up and running with the projects all set are expected, False otherwise,
* @return True if the expected state is achieved within the timeout, False otherwise.
*/
@Throws(IOException::class, CancellationException::class)
private suspend fun doWaitServerState(
private suspend fun waitForState(
isReadyState: Boolean,
timeout: Long = readyTimeout,
timeout: Long,
checkCancelled: (() -> Unit)? = null
): Boolean =
@Suppress("ConvertLongToDuration")
withTimeoutOrNull(timeout * 1000L) {
thisLogger().info(
"Waiting for IDE server on pod '${pod.metadata?.name}' " +
"container '${container.name}' to ${if (isReadyState) "become ready" else "terminate"}; " +
"timeout: ${timeout}s."
)
var pollCount = 0
val refreshFailures = intArrayOf(0)
while (true) {
checkCancelled?.invoke()
if (isServerState(
isReadyState,
checkCancelled,
// Re-resolve pod while waiting for ready so a recycled pod is not missed.
refreshPodBeforeCheck = isReadyState,
refreshFailures,
)
) {
thisLogger().info(
"IDE server on pod '${pod.metadata?.name}' " +
"${if (isReadyState) "is ready" else "terminated"} after ${pollCount * 500}ms."
)
return@withTimeoutOrNull true
}

pollCount++
if (pollCount % 10 == 0) {
thisLogger().debug(
"Still waiting for IDE server on pod '${pod.metadata?.name}' " +
"(${pollCount * 500}ms / ${timeout * 1000}ms)."
)
}
yield()
delay(500L)
}

@Suppress("UNREACHABLE_CODE")
false
} ?: false
): Boolean {
return RemoteIDEServerReadiness(
targetDescription = {
"IDE server on pod '${pod.metadata?.name}' container '${container.name}'"
},
isReady = { cancelled -> getStatus(cancelled).isReady },
refresh = {
pod = findPod()
container = findContainer()
},
).waitFor(isReadyState, timeout, checkCancelled)
}

@Throws(IOException::class)
private fun findPod(): V1Pod {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
/*
* Copyright (c) 2026 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package com.redhat.devtools.gateway.server

import com.intellij.openapi.diagnostic.thisLogger
import com.redhat.devtools.gateway.util.ExponentialBackoff
import com.redhat.devtools.gateway.util.isCancellationException
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.delay
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.coroutines.yield
import java.io.IOException

/**
* Polls a state check with exponential backoff until the expected state is reached
* or the given timeout elapses.
*
* @param targetDescription Human-readable description of the waited target, used in log messages.
* @param isReady Returns whether the server is currently ready. Non-terminal exceptions are
* caught and treated as "not ready"; [CancellationException] and
* [ServerContainerNotFoundException] are rethrown.
* @param refresh Target re-resolution before each readiness probe (only used when waiting
* for ready). Failures are retried with a warning after [REFRESH_FAILURE_WARNING_THRESHOLD]
* consecutive failures. Terminal exceptions are rethrown.
* @param backoff Delay sequence between probes. A successful refresh must NOT reset it:
* while the target is still not ready the delays keep growing (500ms, 1s, 2s, ... capped)
* instead of polling at a constant rate.
*/
class RemoteIDEServerReadiness(
private val targetDescription: () -> String,
private val isReady: suspend (checkCancelled: (() -> Unit)?) -> Boolean,
private val refresh: (() -> Unit)? = null,
private val backoff: ExponentialBackoff = ExponentialBackoff(),
) {
/**
* Waits until [isReady] reports the expected state.
*
* @param isReadyState True if the server becoming ready is expected, false if termination is expected.
* @param timeout Maximum waiting period in seconds.
* @param checkCancelled Optional user-cancellation check invoked before every probe.
* @return True if the expected state is achieved within the timeout, false otherwise.
*/
@Throws(IOException::class, CancellationException::class)
suspend fun waitFor(
isReadyState: Boolean,
timeout: Long,
checkCancelled: (() -> Unit)? = null,
): Boolean =
@Suppress("ConvertLongToDuration")
withTimeoutOrNull(timeout * MILLISECONDS_PER_SECOND) {
logWaitingForState(isReadyState, timeout)
val refreshFailures = intArrayOf(0)
var pollCount = 0
var elapsedMillis = 0L
while (true) {
checkCancelled?.invoke()
// On a transient refresh failure the probe is skipped for this
// iteration, same as the old refreshPodBeforeCheck behavior.
val probeAllowed = skipCheck(isReadyState) || attemptRefresh(refreshFailures)
if (probeAllowed) {
val stateReached = try {
isReady(checkCancelled) == isReadyState
} catch (e: Exception) {
if (e.isCancellationException() || e is ServerContainerNotFoundException) throw e
thisLogger().debug("Failed to check ${targetDescription()} state.", e)
false
}
if (stateReached) {
logStateReached(isReadyState, elapsedMillis)
return@withTimeoutOrNull true
}
}

pollCount++
logStillWaiting(pollCount, elapsedMillis, timeout)
yield()
val delayMillis = backoff.nextDelayMillis()
elapsedMillis += delayMillis
delay(delayMillis)
}

@Suppress("UNREACHABLE_CODE")
false
} ?: false

private fun skipCheck(isReadyState: Boolean): Boolean = !isReadyState || refresh == null

private fun attemptRefresh(refreshFailures: IntArray): Boolean {
val doRefresh = refresh ?: return true
return try {
doRefresh()
refreshFailures[0] = 0
true
} catch (e: Exception) {
if (e.isCancellationException() || e is ServerContainerNotFoundException) throw e
refreshFailures[0]++
thisLogger().debug("Failed to refresh ${targetDescription()} during state check", e)
if (refreshFailures[0] == REFRESH_FAILURE_WARNING_THRESHOLD) {
thisLogger().warn(
"Refresh of ${targetDescription()} has failed ${refreshFailures[0]} consecutive times; " +
"stale references may cause incorrect state checks"
)
}
false
}
}

private fun logWaitingForState(isReadyState: Boolean, timeout: Long) {
thisLogger().info(
"Waiting for ${targetDescription()} to ${if (isReadyState) "become ready" else "terminate"}; " +
"timeout: ${timeout}s."
)
}

private fun logStateReached(isReadyState: Boolean, elapsedMillis: Long) {
thisLogger().info(
"${targetDescription()} ${if (isReadyState) "is ready" else "terminated"} after ${elapsedMillis}ms."
)
}

private fun logStillWaiting(pollCount: Int, elapsedMillis: Long, timeout: Long) {
if (pollCount % STILL_WAITING_LOG_INTERVAL != 0) {
return
}
thisLogger().debug(
"Still waiting for ${targetDescription()} " +
"(${elapsedMillis}ms / ${timeout * MILLISECONDS_PER_SECOND}ms)."
)
}

companion object {
private const val MILLISECONDS_PER_SECOND = 1000L
private const val STILL_WAITING_LOG_INTERVAL = 10

/** Number of consecutive refresh failures before emitting a warning. */
private const val REFRESH_FAILURE_WARNING_THRESHOLD = 10
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Copyright (c) 2026 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package com.redhat.devtools.gateway.util

/**
* Produces a sequence of delays that start at [initialMillis] and double per call,
* capped at [maxMillis].
*/
class ExponentialBackoff(
private val initialMillis: Long = 500,
private val maxMillis: Long = 5000,
) {
private var current = initialMillis.coerceIn(0L, maxMillis)

fun nextDelayMillis(): Long {
val delay = current
current = if (current >= maxMillis / 2) maxMillis else current * 2
return delay
}

fun reset() {
current = initialMillis.coerceIn(0L, maxMillis)
}
}
Loading
Loading