From 9e50ec41be60056ddd3a2f559f378bf3826c0e31 Mon Sep 17 00:00:00 2001 From: witbrock Date: Thu, 27 Aug 2026 10:34:52 -0700 Subject: [PATCH] JVNAUTOSCI-916 enable concurrent scoped turns --- infra/openstack/variables.tf | 2 +- src/backend/db/mongo_client.py | 57 +- src/backend/server/routes/von_routes.py | 1170 ++++++++++++----- .../services/background_task_service.py | 325 ++++- .../services/chat_prompt_queue_service.py | 637 ++++++++- .../conversation_turn_admission_service.py | 489 +++++++ .../services/tool_progress_store_service.py | 30 + .../window_session_context_service.py | 25 +- .../web/von_interface/static/js/apiService.js | 61 +- .../web/von_interface/static/js/chatTab.js | 669 +++++++--- .../static/js/components/orgSelector.js | 31 +- .../paperRecommendationProfilePanel.js | 2 + .../web/von_interface/static/js/domUtils.js | 49 +- .../web/von_interface/static/js/main.js | 112 +- .../von_interface/static/js/settingsPage.js | 40 +- .../static/js/test/chatTab.test.js | 200 ++- .../js/test/sessionScopedStorage.test.js | 55 + .../js/test/userPreferenceBootstrap.test.js | 16 + .../js/test/windowSessionIdentity.test.js | 281 ++++ .../js/utils/governedPublicationScope.js | 11 +- .../js/utils/runtimeIdentityBootstrap.js | 11 +- .../static/js/utils/sessionScopedStorage.js | 62 +- .../js/utils/userPreferenceBootstrap.js | 10 +- .../static/js/utils/windowSessionIdentity.js | 389 ++++++ .../workflowCapabilityStatusCoordinator.js | 23 +- .../web/von_interface/static/js/vontology.js | 5 +- .../static/js/workflowStudioPage.js | 4 +- .../templates/von_interface.html | 2 +- src/workflows/von/main.py | 23 +- tests/backend/test_background_task_service.py | 381 +++++- .../backend/test_chat_prompt_queue_routes.py | 133 +- .../backend/test_chat_prompt_queue_service.py | 276 +++- ...est_conversation_turn_admission_service.py | 489 +++++++ tests/backend/test_tool_progress_liveness.py | 125 +- ...t_von_background_durable_reconciliation.py | 96 +- ...test_von_generate_background_submission.py | 224 +++- .../test_von_route_json_safe_payloads.py | 136 +- ...test_window_session_multi_org_isolation.py | 161 ++- .../apiServiceWindowSessionIdentity.test.js | 91 ++ tests/frontend/chatTabAbort.test.js | 55 +- tests/frontend/chatTaskQueue.test.js | 472 ++++++- .../frontend/runtimeIdentityBootstrap.test.js | 29 + 42 files changed, 6431 insertions(+), 1028 deletions(-) create mode 100644 src/backend/services/conversation_turn_admission_service.py create mode 100644 src/frontend/web/von_interface/static/js/test/windowSessionIdentity.test.js create mode 100644 src/frontend/web/von_interface/static/js/utils/windowSessionIdentity.js create mode 100644 tests/backend/test_conversation_turn_admission_service.py create mode 100644 tests/frontend/apiServiceWindowSessionIdentity.test.js diff --git a/infra/openstack/variables.tf b/infra/openstack/variables.tf index c09424e55..c36d0cb48 100644 --- a/infra/openstack/variables.tf +++ b/infra/openstack/variables.tf @@ -409,7 +409,7 @@ variable "bootstrap_healthcheck_path" { variable "bootstrap_waitress_threads" { type = number description = "VON_WAITRESS_THREADS value written to the managed environment file." - default = 16 + default = 32 validation { condition = var.bootstrap_waitress_threads >= 4 && var.bootstrap_waitress_threads <= 256 diff --git a/src/backend/db/mongo_client.py b/src/backend/db/mongo_client.py index f3a932cb2..8d8ff16f8 100644 --- a/src/backend/db/mongo_client.py +++ b/src/backend/db/mongo_client.py @@ -990,9 +990,9 @@ def _fallback_candidate( ) -def _try_preferred_fallback_candidate() -> ( - tuple[_MongoConnectionCandidate | None, bool] -): +def _try_preferred_fallback_candidate() -> tuple[ + _MongoConnectionCandidate | None, bool +]: """Build, but do not publish, a client for the recent fallback route.""" preferred_kind = _preferred_fallback_kind @@ -2098,6 +2098,57 @@ def _ensure_chat_prompt_queue_indexes(coll: Collection) -> None: ], name="scope_session_status_created_at", ) + if "active_conversation_key_unique" not in existing_indexes: + coll.create_index( + [("active_conversation_key", ASCENDING)], + name="active_conversation_key_unique", + unique=True, + partialFilterExpression={ + "active_conversation_key": {"$exists": True, "$type": "string"} + }, + ) + if "queued_global_slot_unique" not in existing_indexes: + coll.create_index( + [("queued_global_slot", ASCENDING)], + name="queued_global_slot_unique", + unique=True, + partialFilterExpression={"queued_global_slot": {"$exists": True}}, + ) + if "queued_user_slot_unique" not in existing_indexes: + coll.create_index( + [("queued_user_slot", ASCENDING)], + name="queued_user_slot_unique", + unique=True, + partialFilterExpression={"queued_user_slot": {"$exists": True}}, + ) + if "conversation_status_created_queue" not in existing_indexes: + coll.create_index( + [ + ("conversation_key", ASCENDING), + ("status", ASCENDING), + ("created_at", ASCENDING), + ("queue_id", ASCENDING), + ], + name="conversation_status_created_queue", + ) + if "user_status_created_at" not in existing_indexes: + coll.create_index( + [ + ("user_concept_id", ASCENDING), + ("status", ASCENDING), + ("created_at", ASCENDING), + ], + name="user_status_created_at", + ) + if "request_user_namespace" not in existing_indexes: + coll.create_index( + [ + ("client_request_id", ASCENDING), + ("user_concept_id", ASCENDING), + ("namespace", ASCENDING), + ], + name="request_user_namespace", + ) if "updated_at_-1" not in existing_indexes: coll.create_index([("updated_at", DESCENDING)], name="updated_at_-1") diff --git a/src/backend/server/routes/von_routes.py b/src/backend/server/routes/von_routes.py index c0f8ce3c5..6ef65b275 100644 --- a/src/backend/server/routes/von_routes.py +++ b/src/backend/server/routes/von_routes.py @@ -7,6 +7,7 @@ session, send_file, make_response, + g, ) import os import re @@ -61,11 +62,21 @@ from ...services.conversation_turn_memory_context_service import ( merge_conversation_situation_turn_projection, ) +from ...services.conversation_turn_admission_service import ( + SERVER_INSTANCE_ID, + ConversationTurnAdmissionError, + build_conversation_key, + conversation_turn_admission_service, +) from ...services.turn_resource_presentation_service import ( annotate_turn_screen_text, plan_turn_resource_presentation, ) -from ...services.background_task_service import background_task_registry +from ...services.background_task_service import ( + BackgroundTaskCapacityReached, + BackgroundTaskScopeMismatch, + background_task_registry, +) from ...services.workflow_payload_store import is_workflow_payload_blob_ref from ...services.live_request_load import ( decrement_live_turns, @@ -76,6 +87,7 @@ VON_SYSTEM_ID, ) from ...services.window_session_context_service import ( + WindowSessionContextUnavailable, WindowSessionOwnershipError, set_window_organisation, clear_window_organisation, @@ -150,6 +162,7 @@ delete_tool_progress_state, discard_queued_tool_progress_state, fetch_tool_progress_state, + find_tool_progress_scope_keys, queue_tool_progress_state_persistence, ) from ...services.turn_timing_telemetry_service import ( @@ -197,6 +210,97 @@ ) von_bp = Blueprint("von", __name__, template_folder=_TEMPLATE_DIR) +_TURN_ADMISSION_CONTEXT_KEY = "von_conversation_turn_admission" + + +def _release_request_turn_admission( + *, + response: Any | None = None, + exception: BaseException | None = None, +) -> None: + """Release the request-local durable turn fence exactly once.""" + + token = getattr(g, _TURN_ADMISSION_CONTEXT_KEY, None) + if token is None or getattr(token, "released", False): + return + + terminal_status: str | None = None + error_text: str | None = None + if exception is not None: + terminal_status = ( + chat_prompt_queue_service.STATUS_CANCELLED + if isinstance(exception, CancellationRequested) + else chat_prompt_queue_service.STATUS_FAILED + ) + error_text = str(exception)[:4_000] + elif response is not None: + terminal_status = chat_prompt_queue_service.STATUS_COMPLETED + status_code = int(getattr(response, "status_code", 200) or 200) + payload = None + try: + payload = response.get_json(silent=True) + except Exception: + payload = None + if isinstance(payload, Mapping) and payload.get("success") is False: + projected_status = str(payload.get("terminal_status") or "").lower() + terminal_status = ( + chat_prompt_queue_service.STATUS_CANCELLED + if projected_status == "cancelled" + else chat_prompt_queue_service.STATUS_FAILED + ) + error_text = str( + payload.get("error") or payload.get("detail") or projected_status + )[:4_000] + elif status_code >= 400: + terminal_status = chat_prompt_queue_service.STATUS_FAILED + error_text = f"/von/generate returned HTTP {status_code}" + else: + terminal_status = getattr(token, "pending_terminal_status", None) + error_text = getattr(token, "pending_terminal_error", None) + + # Teardown without a response is not evidence of successful completion. + # It may be retrying a durable finish whose first write outcome was + # unknown, so retain the exact disposition computed by after_request. + if terminal_status is None: + return + pending_terminal_status = getattr(token, "pending_terminal_status", None) + if pending_terminal_status is None: + token.pending_terminal_status = terminal_status + token.pending_terminal_error = error_text + else: + # The first terminal observation is final for this exact attempt. An + # error surfaced later during response finalisation or teardown must + # not rewrite a completed/cancelled/failed durable disposition. + terminal_status = pending_terminal_status + error_text = getattr(token, "pending_terminal_error", None) + + try: + conversation_turn_admission_service.release( + token, + status=terminal_status, + error=error_text, + ) + except Exception: + current_app.logger.exception( + "[turn_admission] Failed to release queue_id=%s request_id=%s", + getattr(token, "queue_id", None), + getattr(token, "client_request_id", None), + ) + + +@von_bp.after_request +def _release_turn_admission_after_response(response: Any) -> Any: + _release_request_turn_admission(response=response) + token = getattr(g, _TURN_ADMISSION_CONTEXT_KEY, None) + if token is not None and getattr(token, "queue_id", None): + response.headers.setdefault("X-Von-Prompt-Queue-ID", token.queue_id) + return response + + +@von_bp.teardown_request +def _release_turn_admission_after_exception(exception: BaseException | None) -> None: + _release_request_turn_admission(exception=exception) + def _json_safe_response_payload(value: Any, *, _seen: set[int] | None = None) -> Any: """Clone arbitrary route payloads into structures Flask can JSON-encode.""" @@ -655,9 +759,14 @@ def _append_selected_workflow_execution_event( def _get_current_chat_prompt_queue_scope() -> dict[str, str | None] | tuple[Any, int]: try: - from ...security.access_control import get_effective_user_concept_id + from ...security.access_control import ( + LEGACY_IDENTITY_HEADER_ACTOR_SOURCE, + get_effective_user_concept_id_with_source, + ) - user_concept_id = get_effective_user_concept_id() + user_concept_id, actor_source = get_effective_user_concept_id_with_source() + if actor_source == LEGACY_IDENTITY_HEADER_ACTOR_SOURCE: + user_concept_id = None except Exception: user_concept_id = session.get("user_concept_id") @@ -674,17 +783,30 @@ def _get_current_chat_prompt_queue_scope() -> dict[str, str | None] | tuple[Any, ) window_session_id = request.headers.get(_WINDOW_SESSION_HEADER_NAME) - effective_context = get_effective_context( - window_session_id, - dict(session), - user_concept_id.strip(), - ) - organisation_concept_id = ( - effective_context.get("organisation_id") - or session.get("org_id") - or session.get("organisation_concept_id") - ) - namespace = effective_context.get("namespace") or session.get("namespace") + try: + effective_context = get_effective_context( + window_session_id, + dict(session), + user_concept_id.strip(), + require_known_window=bool(window_session_id), + ) + except WindowSessionContextUnavailable: + return ( + jsonify( + { + "success": False, + "error": "Window organisation context must be rebound", + "error_code": "window_context_unavailable", + "retryable": True, + } + ), + 409, + ) + # `get_effective_context` has already applied the legacy no-window fallback. + # Do not use truthiness fallbacks here: an explicit personal window has an + # authoritative null organisation and must not inherit another tab's org. + organisation_concept_id = effective_context.get("organisation_id") + namespace = effective_context.get("namespace") return chat_prompt_queue_service.build_queue_scope( user_concept_id=user_concept_id.strip(), organisation_concept_id=organisation_concept_id, @@ -725,6 +847,19 @@ def _chat_prompt_queue_error_response( queue_id: str | None = None, scope: Mapping[str, Any] | None = None, ): + if isinstance(exc, chat_prompt_queue_service.ChatPromptQueueCapacityReached): + response = jsonify( + { + "success": False, + "error": str(exc), + "error_code": "foreground_queue_capacity_reached", + "limit_kind": exc.limit_kind, + "retryable": True, + "retry_after_seconds": 1, + } + ) + response.headers["Retry-After"] = "1" + return response, 429 if isinstance(exc, chat_prompt_queue_service.InvalidChatPromptQueueInput): return ( jsonify( @@ -736,6 +871,18 @@ def _chat_prompt_queue_error_response( ), 400, ) + if isinstance(exc, chat_prompt_queue_service.ConversationTurnAlreadyActive): + return ( + jsonify( + { + "success": False, + "error": str(exc), + "error_code": "conversation_turn_active", + "retryable": True, + } + ), + 409, + ) if isinstance(exc, chat_prompt_queue_service.ChatPromptQueueRecordNotFound): current_app.logger.warning( "Chat prompt queue transition miss action=%s queue_id=%s code=%s details=%s scope=%s", @@ -793,7 +940,13 @@ def list_chat_prompt_queue_route(): return scope try: records = chat_prompt_queue_service.list_queue_visibility_records(scope=scope) - return jsonify({"success": True, **records}) + return jsonify( + { + "success": True, + **records, + "turn_admission": conversation_turn_admission_service.snapshot(), + } + ) except Exception as exc: return _chat_prompt_queue_error_response(exc, action="list", scope=scope) @@ -804,22 +957,42 @@ def create_chat_prompt_queue_route(): if isinstance(scope, tuple): return scope payload = _chat_prompt_queue_payload() - status = ( - chat_prompt_queue_service.STATUS_IN_PROGRESS - if payload.get("status") == chat_prompt_queue_service.STATUS_IN_PROGRESS - else chat_prompt_queue_service.STATUS_QUEUED - ) - source = ( - "active" if status == chat_prompt_queue_service.STATUS_IN_PROGRESS else "queued" - ) try: + raw_session_id = payload.get("session_id") + session_id = ( + raw_session_id.strip() + if isinstance(raw_session_id, str) and raw_session_id.strip() + else None + ) + conversation_key = None + if session_id: + actor_user_id = str(scope.get("user_concept_id") or "") + owner_user_id, shared_invite = _resolve_shared_conversation_owner( + user_concept_id=actor_user_id, + session_id=session_id, + ) + history_user_id = owner_user_id or actor_user_id + history_namespace = _canonical_conversation_history_namespace( + owner_user_id=history_user_id, + actor_namespace=scope.get("namespace"), + shared_invite=shared_invite, + ) + if history_user_id and history_namespace: + conversation_key = build_conversation_key( + owner_user_id=history_user_id, + history_namespace=history_namespace, + conversation_session_id=session_id, + ) record = chat_prompt_queue_service.create_queue_record( scope=scope, prompt_raw=payload.get("prompt_raw"), - session_id=payload.get("session_id"), + session_id=session_id, session_name=payload.get("session_name"), - status=status, - source=source, + status=chat_prompt_queue_service.STATUS_QUEUED, + source="queued", + client_request_id=payload.get("client_request_id"), + attempt_id=payload.get("attempt_id"), + conversation_key=conversation_key, ) return jsonify({"success": True, "item": record}), 201 except Exception as exc: @@ -833,12 +1006,43 @@ def update_chat_prompt_queue_route(queue_id: str): return scope payload = _chat_prompt_queue_payload() try: + update_kwargs: dict[str, Any] = {} + if "session_id" in payload: + raw_session_id = payload.get("session_id") + session_id = ( + raw_session_id.strip() + if isinstance(raw_session_id, str) and raw_session_id.strip() + else None + ) + conversation_key = None + if session_id: + actor_user_id = str(scope.get("user_concept_id") or "") + owner_user_id, shared_invite = _resolve_shared_conversation_owner( + user_concept_id=actor_user_id, + session_id=session_id, + ) + history_user_id = owner_user_id or actor_user_id + history_namespace = _canonical_conversation_history_namespace( + owner_user_id=history_user_id, + actor_namespace=scope.get("namespace"), + shared_invite=shared_invite, + ) + if history_user_id and history_namespace: + conversation_key = build_conversation_key( + owner_user_id=history_user_id, + history_namespace=history_namespace, + conversation_session_id=session_id, + ) + update_kwargs = { + "session_id": session_id, + "conversation_key": conversation_key, + } record = chat_prompt_queue_service.update_queued_record( scope=scope, queue_id=queue_id, prompt_raw=payload.get("prompt_raw"), - session_id=payload.get("session_id"), session_name=payload.get("session_name"), + **update_kwargs, ) return jsonify({"success": True, "item": record}) except Exception as exc: @@ -875,19 +1079,17 @@ def claim_chat_prompt_queue_route(queue_id: str): scope = _get_current_chat_prompt_queue_scope() if isinstance(scope, tuple): return scope - try: - record = chat_prompt_queue_service.claim_queue_record( - scope=scope, - queue_id=queue_id, - ) - return jsonify({"success": True, "item": record}) - except Exception as exc: - return _chat_prompt_queue_error_response( - exc, - action="claim", - queue_id=queue_id, - scope=scope, - ) + return ( + jsonify( + { + "success": False, + "error": "Queue claims are owned by the server turn lifecycle", + "error_code": "queue_claim_route_retired", + "retryable": False, + } + ), + 410, + ) @von_bp.route("/api/chat_prompt_queue//requeue", methods=["POST"]) @@ -899,6 +1101,7 @@ def requeue_chat_prompt_queue_route(queue_id: str): record = chat_prompt_queue_service.requeue_prompt_record( scope=scope, queue_id=queue_id, + current_server_instance_id=SERVER_INSTANCE_ID, ) return jsonify({"success": True, "item": record}) except Exception as exc: @@ -3761,11 +3964,7 @@ def _tool_call_descriptor( ), } - if ( - isinstance(session_id, str) - and session_id.strip() - and delegated_actor_user_id - ): + if isinstance(session_id, str) and session_id.strip() and delegated_actor_user_id: conversation_ref = build_conversation_scope_binding( chat_session_id=session_id, history_owner_user_id=history_owner_user_id, @@ -3913,11 +4112,7 @@ def _build_turn_execution_diagnostics( "failure_kind", ) diagnostic_events = [ - { - key: entry[key] - for key in neutral_event_fields - if key in entry - } + {key: entry[key] for key in neutral_event_fields if key in entry} for entry in raw_events if isinstance(entry, dict) ][-_TURN_EXECUTION_DIAGNOSTICS_EVENT_LIMIT:] @@ -4373,6 +4568,32 @@ def _normalise_tool_progress_window_session_id(value: Any) -> str | None: return clean_value +def _build_authenticated_tool_progress_scope_key( + *, + user_concept_id: str, + organisation_concept_id: str | None, + namespace: str, +) -> str: + """Build a content-free key for one exact trusted actor/org namespace.""" + + canonical = chat_prompt_queue_service.build_queue_scope( + user_concept_id=user_concept_id, + organisation_concept_id=organisation_concept_id, + namespace=namespace, + ) + canonical_namespace = _progress_str(canonical.get("namespace")) + if not canonical_namespace: + raise ValueError("exact namespace is required for live progress") + material = "\x1f".join( + ( + str(canonical.get("user_concept_id") or ""), + str(canonical.get("organisation_concept_id") or ""), + canonical_namespace, + ) + ) + return "actor-scope:" + hashlib.sha256(material.encode("utf-8")).hexdigest() + + def _get_tool_progress_scope_key() -> str: """Return a stable scope key for tool-progress lookup. @@ -5857,39 +6078,62 @@ def get_generation_progress(request_id: str): ): return jsonify({"error": "Invalid request_id"}), 400 + try: + from ...security.access_control import ( + LEGACY_IDENTITY_HEADER_ACTOR_SOURCE, + get_effective_user_concept_id_with_source, + ) + + user_concept_id, actor_source = get_effective_user_concept_id_with_source() + if actor_source == LEGACY_IDENTITY_HEADER_ACTOR_SOURCE: + user_concept_id = None + except Exception: + user_concept_id = session.get("user_concept_id") + if not isinstance(user_concept_id, str) or not user_concept_id.strip(): + return jsonify({"error": "progress_authentication_required"}), 401 + window_session_id = _normalise_tool_progress_window_session_id( request.headers.get(_WINDOW_SESSION_HEADER_NAME) ) - header_user_concept_id = _normalise_concept_id( - request.headers.get("X-User-Concept-ID") - ) or _progress_str(request.headers.get("X-User-Concept-ID")) - - scope_key = "" - resolved_user_concept_id = header_user_concept_id - anonymous_session_id = None - - # Keep live progress polling as header-first as possible so browser turns do - # not depend on Flask-session reads while /von/generate is still active. - if resolved_user_concept_id: - scope_key = f"user:{resolved_user_concept_id}" - elif window_session_id: - scope_key = f"{_TOOL_PROGRESS_WINDOW_SCOPE_PREFIX}{window_session_id}" - else: - scope_key = _get_tool_progress_scope_key() - if scope_key.startswith("user:"): - resolved_user_concept_id = _progress_str(scope_key[len("user:") :]) - else: - resolved_user_concept_id = _progress_str(session.get("user_concept_id")) - anonymous_session_id = _progress_str(session.get("tool_progress_scope")) + if not window_session_id: + return jsonify({"error": "progress_window_scope_required"}), 409 + try: + effective = get_effective_context( + window_session_id, + dict(session), + user_concept_id.strip(), + require_known_window=True, + ) + namespace = _progress_str(effective.get("namespace")) + if not namespace: + raise WindowSessionContextUnavailable("window_session_context_unavailable") + scope_key = _build_authenticated_tool_progress_scope_key( + user_concept_id=user_concept_id.strip(), + organisation_concept_id=_normalise_concept_id( + effective.get("organisation_id") + ), + namespace=namespace, + ) + except WindowSessionContextUnavailable: + return jsonify({"error": "progress_window_scope_unavailable"}), 409 - state, resolved_scope_key = _resolve_tool_progress_state_from_scope_candidates( - request_id=request_id.strip(), - explicit_scope_key=scope_key, - user_concept_id=resolved_user_concept_id, - window_session_id=window_session_id, - anonymous_session_id=anonymous_session_id, - ) + clean_request_id = request_id.strip() + state = _get_tool_progress(scope_key, clean_request_id) if not state: + with _TOOL_PROGRESS_LOCK: + known_scopes = { + candidate_scope + for candidate_scope, candidate_request_id in _TOOL_PROGRESS + if candidate_request_id == clean_request_id + } + known_scopes.update( + find_tool_progress_scope_keys(request_id=clean_request_id, limit=4) + ) + if any( + candidate.startswith("actor-scope:") and candidate != scope_key + for candidate in known_scopes + ): + return jsonify({"error": "progress_scope_mismatch"}), 404 try: show_tool_use_progress = bool(get_show_tool_use_during_thinking()) except Exception: @@ -5909,13 +6153,6 @@ def get_generation_progress(request_id: str): ) serialised_state = _serialise_tool_progress_state(state) - if ( - isinstance(resolved_scope_key, str) - and resolved_scope_key - and resolved_scope_key != scope_key - ): - serialised_state["resolved_scope_key"] = resolved_scope_key - serialised_state["progress_source"] = "alternate_scope_fallback" return jsonify(_json_safe_response_payload(serialised_state)), 200 @@ -6216,13 +6453,31 @@ def _build_durable_turn_background_result(instance: Any) -> dict[str, Any]: return result_payload -def _find_terminal_durable_turn_instance(task_id: str) -> Any | None: +def _durable_instance_matches_task_scope( + instance: Any, + scope: Mapping[str, Any], +) -> bool: + return bool( + getattr(instance, "user_id", None) == scope.get("user_concept_id") + and getattr(instance, "org_id", None) == scope.get("organisation_concept_id") + and getattr(instance, "namespace", None) == scope.get("namespace") + ) + + +def _find_terminal_durable_turn_instance( + task_id: str, + *, + scope: Mapping[str, Any], +) -> Any | None: try: manager = get_instance_manager() instances = manager.list_instances( workflow_id=CONVERSATION_TURN_EXECUTION_WORKFLOW_ID, source_event_type="conversation_turn", source_event_id=task_id, + user_id=scope.get("user_concept_id"), + org_id=scope.get("organisation_concept_id"), + namespace=scope.get("namespace"), status=( WorkflowInstanceStatus.COMPLETED, WorkflowInstanceStatus.FAILED, @@ -6233,6 +6488,8 @@ def _find_terminal_durable_turn_instance(task_id: str) -> Any | None: if not instances: return None compact_instance = instances[0] + if not _durable_instance_matches_task_scope(compact_instance, scope): + return None instance_id = _normalise_non_empty_text( getattr(compact_instance, "instance_id", None) ) @@ -6250,7 +6507,14 @@ def _find_terminal_durable_turn_instance(task_id: str) -> Any | None: ) else: if hydrated_instance is not None: - return hydrated_instance + return ( + hydrated_instance + if _durable_instance_matches_task_scope( + hydrated_instance, + scope, + ) + else None + ) return compact_instance except Exception as exc: current_app.logger.warning( @@ -6261,7 +6525,12 @@ def _find_terminal_durable_turn_instance(task_id: str) -> Any | None: return None -def _reconcile_background_task_from_durable_turn(task_id: str, status: Any) -> Any: +def _reconcile_background_task_from_durable_turn( + task_id: str, + status: Any, + *, + scope: Mapping[str, Any], +) -> Any: if status is not None and getattr(status, "status", None) == "completed": progress = getattr(status, "progress", None) if isinstance(progress, Mapping) and _normalise_non_empty_text( @@ -6269,7 +6538,7 @@ def _reconcile_background_task_from_durable_turn(task_id: str, status: Any) -> A ): return status - instance = _find_terminal_durable_turn_instance(task_id) + instance = _find_terminal_durable_turn_instance(task_id, scope=scope) if instance is None: return status @@ -6302,14 +6571,60 @@ def _reconcile_background_task_from_durable_turn(task_id: str, status: Any) -> A error = getattr(instance, "error", None) if task_status != "completed" else None marker = getattr(background_task_registry, "mark_terminal_external", None) if callable(marker): - return marker( + try: + reconciled = marker( + task_id, + status=task_status, + result=result, + error=error, + progress=progress, + session_id=result.get("session_id"), + user_id=scope.get("user_concept_id"), + organisation_id=scope.get("organisation_concept_id"), + namespace=scope.get("namespace"), + ) + except BackgroundTaskScopeMismatch: + current_app.logger.warning( + "[background_task] Rejected cross-scope durable reconciliation " + "for task %s", + task_id, + ) + return status + if not all( + ( + getattr(reconciled, "user_id", None) == scope.get("user_concept_id"), + getattr(reconciled, "organisation_id", None) + == scope.get("organisation_concept_id"), + getattr(reconciled, "namespace", None) == scope.get("namespace"), + ) + ): + return status + return reconciled + return status + + +def _get_background_task_status_for_scope( + task_id: str, + scope: Mapping[str, Any], +) -> Any | None: + getter = getattr(background_task_registry, "get_task_status_for_scope", None) + if callable(getter): + return getter( task_id, - status=task_status, - result=result, - error=error, - progress=progress, - session_id=result.get("session_id"), + user_id=scope.get("user_concept_id"), + organisation_id=scope.get("organisation_concept_id"), + namespace=scope.get("namespace"), ) + status = background_task_registry.get_task_status(task_id) + if status is None: + return None + if ( + getattr(status, "user_id", None) != scope.get("user_concept_id") + or getattr(status, "organisation_id", None) + != scope.get("organisation_concept_id") + or getattr(status, "namespace", None) != scope.get("namespace") + ): + return None return status @@ -6325,9 +6640,16 @@ def get_task_status(task_id: str): if not isinstance(task_id, str) or not task_id.strip() or len(task_id) > 200: return jsonify({"error": "Invalid task_id"}), 400 + scope = _get_current_chat_prompt_queue_scope() + if isinstance(scope, tuple): + return scope task_id_clean = task_id.strip() - status = background_task_registry.get_task_status(task_id_clean) - status = _reconcile_background_task_from_durable_turn(task_id_clean, status) + status = _get_background_task_status_for_scope(task_id_clean, scope) + status = _reconcile_background_task_from_durable_turn( + task_id_clean, + status, + scope=scope, + ) if status is None: return jsonify({"error": "Task not found", "task_id": task_id}), 404 @@ -6436,6 +6758,8 @@ def _mark_background_generate_completed_if_ready( request_id: str, session_id: str | None, user_id: str | None, + organisation_id: str | None, + namespace: str | None, response_text: str | None, ) -> None: """Expose a completed background generate result before slow persistence. @@ -6474,6 +6798,8 @@ def _mark_background_generate_completed_if_ready( progress=_json_safe_response_payload(progress), session_id=session_id, user_id=user_id, + organisation_id=organisation_id, + namespace=namespace, ) except Exception: current_app.logger.debug( @@ -6606,9 +6932,16 @@ def get_task_result(task_id: str): if not isinstance(task_id, str) or not task_id.strip() or len(task_id) > 200: return jsonify({"error": "Invalid task_id"}), 400 + scope = _get_current_chat_prompt_queue_scope() + if isinstance(scope, tuple): + return scope task_id_clean = task_id.strip() - status = background_task_registry.get_task_status(task_id_clean) - status = _reconcile_background_task_from_durable_turn(task_id_clean, status) + status = _get_background_task_status_for_scope(task_id_clean, scope) + status = _reconcile_background_task_from_durable_turn( + task_id_clean, + status, + scope=scope, + ) if status is None: return jsonify({"error": "Task not found", "task_id": task_id}), 404 @@ -6672,7 +7005,38 @@ def cancel_task(task_id: str): if not isinstance(task_id, str) or not task_id.strip() or len(task_id) > 200: return jsonify({"error": "Invalid task_id"}), 400 - success = background_task_registry.request_cancellation(task_id.strip()) + scope = _get_current_chat_prompt_queue_scope() + if isinstance(scope, tuple): + return scope + task_id_clean = task_id.strip() + status = _get_background_task_status_for_scope(task_id_clean, scope) + if status is None: + return ( + jsonify( + { + "error": "Cannot cancel task", + "task_id": task_id, + "detail": "Task not found or already completed", + } + ), + 404, + ) + cancel_for_scope = getattr( + background_task_registry, + "request_cancellation_for_scope", + None, + ) + if callable(cancel_for_scope): + success = cancel_for_scope( + task_id_clean, + user_id=scope.get("user_concept_id"), + organisation_id=scope.get("organisation_concept_id"), + namespace=scope.get("namespace"), + ) + else: + success = bool(status) and background_task_registry.request_cancellation( + task_id_clean + ) if not success: return ( jsonify( @@ -6685,6 +7049,22 @@ def cancel_task(task_id: str): 404, ) + queue_id = _normalise_non_empty_text(getattr(status, "queue_id", None)) + if queue_id: + try: + chat_prompt_queue_service.cancel_prompt_record( + scope=scope, + queue_id=queue_id, + ) + except chat_prompt_queue_service.ConversationTurnAlreadyActive: + # The running generate request owns this fence and observes the + # cancellation flag; its teardown will terminalise the exact + # attempt as cancelled. + pass + except chat_prompt_queue_service.ChatPromptQueueRecordNotFound: + # The exact attempt may already have reached a terminal state. + pass + return ( jsonify( {"success": True, "task_id": task_id, "message": "Cancellation requested"} @@ -6700,7 +7080,7 @@ def list_tasks(): Query params: status: Filter by status (pending, running, completed, failed, cancelled) session_id: Filter by session ID (Phase 4) - user_id: Filter by user ID (Phase 4) + Actor and organisation scope always come from the authenticated request. Returns: 200: List of task status dicts @@ -6715,15 +7095,37 @@ def list_tasks(): ): return jsonify({"error": "Invalid status filter"}), 400 + scope = _get_current_chat_prompt_queue_scope() + if isinstance(scope, tuple): + return scope session_id = request.args.get("session_id") - user_id = request.args.get("user_id") - tasks = background_task_registry.list_tasks( - status_filter=status_filter, - session_id=session_id, - user_id=user_id, + scoped_list = getattr(background_task_registry, "list_tasks_for_scope", None) + if callable(scoped_list): + tasks = scoped_list( + status_filter=status_filter, + session_id=session_id, + user_id=scope.get("user_concept_id"), + organisation_id=scope.get("organisation_concept_id"), + namespace=scope.get("namespace"), + ) + else: + tasks = background_task_registry.list_tasks( + status_filter=status_filter, + session_id=session_id, + user_id=scope.get("user_concept_id"), + organisation_id=scope.get("organisation_concept_id"), + namespace=scope.get("namespace"), + ) + return ( + jsonify( + { + "tasks": tasks, + "turn_admission": conversation_turn_admission_service.snapshot(), + } + ), + 200, ) - return jsonify({"tasks": tasks}), 200 # ----------------- End Background Tasks ----------------- @@ -6835,9 +7237,7 @@ def _serialise_tool_invocations_for_llm_debug( transport_metadata = raw_invocation.get("transport") if isinstance(transport_metadata, Mapping): - entry["transport"] = _sanitise_diagnostic_export_payload( - transport_metadata - ) + entry["transport"] = _sanitise_diagnostic_export_payload(transport_metadata) serialised.append(entry) @@ -6896,9 +7296,7 @@ def _serialise_tool_invocations_for_turn_execution_record( transport_metadata = raw_invocation.get("transport") if isinstance(transport_metadata, Mapping): - entry["transport"] = _sanitise_diagnostic_export_payload( - transport_metadata - ) + entry["transport"] = _sanitise_diagnostic_export_payload(transport_metadata) evidence = raw_invocation.get("evidence") if isinstance(evidence, Mapping): @@ -7457,9 +7855,10 @@ def _finalise_llm_debug_info( if isinstance(response_text, str) else str(llm_debug_info.get("response") or "") ) - terminal_status = _progress_str( - llm_interaction_payload.get("ordinary_turn_terminal_status") - ) or "not_reported" + terminal_status = ( + _progress_str(llm_interaction_payload.get("ordinary_turn_terminal_status")) + or "not_reported" + ) outcome_report: Mapping[str, Any] | None = None outcome_report_sources: list[Any] = [*aux_llm_calls_payload] if isinstance(diagnostics_payload, Mapping): @@ -7471,8 +7870,7 @@ def _finalise_llm_debug_info( continue if ( entry.get("type") == "adaptive_turn_effect_outcome_report" - and entry.get("schema_version") - == "adaptive_turn_effect_outcome_report.v1" + and entry.get("schema_version") == "adaptive_turn_effect_outcome_report.v1" ): outcome_report = entry break @@ -7484,9 +7882,7 @@ def _finalise_llm_debug_info( response_authority = _progress_str(llm_debug_info.get("response_authority")) if response_authority is None and isinstance(outcome_report, Mapping): - response_authority = _progress_str( - outcome_report.get("response_authority") - ) + response_authority = _progress_str(outcome_report.get("response_authority")) report_facts = ( outcome_report.get("facts") if isinstance(outcome_report, Mapping) @@ -7533,9 +7929,7 @@ def _finalise_llm_debug_info( "record_kind": "observational", "request_id": llm_debug_info.get("request_id"), "session_id": session_id, - "interaction_timestamp_utc": llm_debug_info.get( - "interaction_timestamp_utc" - ), + "interaction_timestamp_utc": llm_debug_info.get("interaction_timestamp_utc"), "actor": { "actor_concept_id": resolved_actor_concept_id, "user_concept_id": user_id, @@ -7553,7 +7947,9 @@ def _finalise_llm_debug_info( ), }, "applied_prompt_snapshot": applied_prompt_snapshot_payload, - "llm_calls": [dict(item) for item in llm_calls_payload if isinstance(item, Mapping)], + "llm_calls": [ + dict(item) for item in llm_calls_payload if isinstance(item, Mapping) + ], "llm_usage_cost_summary": llm_usage_cost_summary, "tool_invocations": [ dict(item) @@ -7564,12 +7960,12 @@ def _finalise_llm_debug_info( dict(tool_observation_ledger) if int(tool_observation_ledger.get("observation_count") or 0) > 0 else None - ), - "search_evidence": list(search_evidence_payload or []), - "evidence_index": evidence_index, - "turn_execution_diagnostics": ( - dict(diagnostics_payload) - if isinstance(diagnostics_payload, Mapping) + ), + "search_evidence": list(search_evidence_payload or []), + "evidence_index": evidence_index, + "turn_execution_diagnostics": ( + dict(diagnostics_payload) + if isinstance(diagnostics_payload, Mapping) else None ), "code_version": code_version_details, @@ -9453,19 +9849,26 @@ def _load_conversation_session_state_fail_soft( request_id, exc, ) - return [], None, None, 0, [], { - "history_offset": 0, - "history_truncated": False, - "conversation_observation_state": { - "schema_version": "conversation_observation_state.v1", - "retained_count": 0, - "total_count": 0, - "omitted_count": 0, - "retention_limit": ( - chat_history_service.CONVERSATION_OBSERVATION_MAX_ITEMS - ), + return ( + [], + None, + None, + 0, + [], + { + "history_offset": 0, + "history_truncated": False, + "conversation_observation_state": { + "schema_version": "conversation_observation_state.v1", + "retained_count": 0, + "total_count": 0, + "omitted_count": 0, + "retention_limit": ( + chat_history_service.CONVERSATION_OBSERVATION_MAX_ITEMS + ), + }, }, - } + ) def _persist_conversation_situation_fail_soft( @@ -9567,25 +9970,19 @@ def _read_conversation_carrier_after_persistence_fail_soft( if not isinstance(session_state, Mapping): raise RuntimeError("canonical conversation session state was not found") - conversation_situation, _, _ = ( - _normalise_conversation_situation_descriptor( - session_state.get("conversation_situation") - ) + conversation_situation, _, _ = _normalise_conversation_situation_descriptor( + session_state.get("conversation_situation") ) observations = ( [ dict(observation) - for observation in session_state.get( - "conversation_observations", [] - ) + for observation in session_state.get("conversation_observations", []) if isinstance(observation, Mapping) ] if isinstance(session_state.get("conversation_observations"), list) else [] ) - raw_observation_state = session_state.get( - "conversation_observation_state" - ) + raw_observation_state = session_state.get("conversation_observation_state") observation_state = ( dict(raw_observation_state) if isinstance(raw_observation_state, Mapping) @@ -10062,6 +10459,8 @@ def _submit_generate_background_request( request_headers: Mapping[str, Any], session_snapshot: Mapping[str, Any], request_id: str, + actor_scope: Mapping[str, Any], + conversation_session_id: str | None, ): background_payload = dict(request_data) if isinstance(request_data, Mapping) else {} background_payload["background"] = False @@ -10070,29 +10469,37 @@ def _submit_generate_background_request( background_payload["background_progress"] = True background_headers: dict[str, str] = {} - for header_name in ( - "X-Von-Window-Session", - "X-User-Concept-ID", - "X-User-Client-ID", - ): - header_value = request_headers.get(header_name) - if isinstance(header_value, str) and header_value.strip(): - background_headers[header_name] = header_value.strip() - - background_session_id = ( - str(session_snapshot.get("session_id")).strip() - if session_snapshot.get("session_id") - else None + # Do not replay the mutable browser-window selector as task authority. The + # exact authenticated scope resolved at submission is frozen into the + # server-side session snapshot below. + del request_headers + + background_session_id = conversation_session_id + background_queue_id = _normalise_non_empty_text( + background_payload.get("prompt_queue_id") ) - background_user_id = ( - str(session_snapshot.get("user_concept_id")).strip() - if session_snapshot.get("user_concept_id") - else ( - str(session_snapshot.get("user_id")).strip() - if session_snapshot.get("user_id") - else None - ) + background_attempt_id = _normalise_non_empty_text( + background_payload.get("attempt_id") ) + background_user_id = _normalise_non_empty_text(actor_scope.get("user_concept_id")) + background_org_id = _normalise_non_empty_text( + actor_scope.get("organisation_concept_id") + ) + background_namespace = _normalise_non_empty_text(actor_scope.get("namespace")) + background_role = _normalise_non_empty_text(actor_scope.get("role_in_org")) + frozen_session_snapshot = dict(session_snapshot) + frozen_session_snapshot["user_concept_id"] = background_user_id + frozen_session_snapshot["namespace"] = background_namespace + if background_org_id: + frozen_session_snapshot["organisation_concept_id"] = background_org_id + frozen_session_snapshot["org_id"] = background_org_id + else: + frozen_session_snapshot.pop("organisation_concept_id", None) + frozen_session_snapshot.pop("org_id", None) + if background_role: + frozen_session_snapshot["role_in_org"] = background_role + else: + frozen_session_snapshot.pop("role_in_org", None) def _run_generate_request_in_background() -> dict[str, Any]: with app.test_request_context( @@ -10102,17 +10509,51 @@ def _run_generate_request_in_background() -> dict[str, Any]: headers=background_headers, ): session.clear() - session.update(session_snapshot) + session.update(frozen_session_snapshot) session.modified = True - return _normalise_background_generate_result(generate()) - - task_status = background_task_registry.submit_task( - task_id=request_id, - callable=_run_generate_request_in_background, - progress_callback=None, - session_id=background_session_id, - user_id=background_user_id, - ) + try: + generated = make_response(generate()) + _release_request_turn_admission(response=generated) + return _normalise_background_generate_result(generated) + except BaseException as exc: + _release_request_turn_admission(exception=exc) + raise + + try: + task_status = background_task_registry.submit_task( + task_id=request_id, + callable=_run_generate_request_in_background, + progress_callback=None, + session_id=background_session_id, + user_id=background_user_id, + organisation_id=background_org_id, + namespace=background_namespace, + queue_id=background_queue_id, + client_request_id=request_id, + attempt_id=background_attempt_id, + ) + except BackgroundTaskCapacityReached as exc: + response = jsonify( + { + "error": "background_task_capacity_reached", + "detail": str(exc), + "retryable": True, + "retry_after_seconds": 1, + } + ) + response.headers["Retry-After"] = "1" + return response, 429 + except ValueError: + return ( + jsonify( + { + "error": "duplicate_background_task_id", + "detail": "This background request identifier is already in use.", + "retryable": False, + } + ), + 409, + ) return ( jsonify( @@ -10294,11 +10735,14 @@ def _build_focal_conversation_runtime_envelope( item = by_id[concept_id] excerpts: list[str] = [] try: - for text_item in get_texts_for_concept( - concept_id, - limit=8, - context_view="actor_effective", - ) or []: + for text_item in ( + get_texts_for_concept( + concept_id, + limit=8, + context_view="actor_effective", + ) + or [] + ): text = text_item.get("text") if isinstance(text_item, Mapping) else None if not isinstance(text, str) or not text.strip(): continue @@ -10421,15 +10865,6 @@ def generate(): # pyright: ignore[reportGeneralTypeIssues] 400, ) - if background_mode: - return _submit_generate_background_request( - app=current_app._get_current_object(), - request_data=data if isinstance(data, Mapping) else None, - request_headers=request.headers, - session_snapshot=dict(session), - request_id=request_id, - ) - request_start_perf = time.perf_counter() turn_timing_recorder = TurnTimingRecorder(request_id=request_id) progress_heartbeat_stop_event: threading.Event | None = None @@ -10438,7 +10873,9 @@ def generate(): # pyright: ignore[reportGeneralTypeIssues] datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") ) - progress_scope_key = _get_tool_progress_bootstrap_scope_key() + # Live progress is not published until authentication and the exact window + # organisation/namespace have been resolved below. + progress_scope_key = "" progress_mirror_scope_keys: list[str] = [] request_window_session_id = request.headers.get(_WINDOW_SESSION_HEADER_NAME) show_tool_use_progress = False @@ -10530,15 +10967,6 @@ def _check_background_cancellation(subtask: str) -> None: raise CancellationRequested(task_id=background_task_id) progress_goal_label = _build_progress_goal_label(prompt_text=prompt_text) - if project_live_progress: - _register_tool_progress_scope_aliases( - request_id=request_id, - primary_scope_key=progress_scope_key, - mirror_scope_keys=progress_mirror_scope_keys, - window_session_id=request_window_session_id, - anonymous_session_id=_progress_str(session.get("tool_progress_scope")), - ) - if progress_updates_enabled: _emit_generate_progress( _build_request_initialising_tool_progress_payload( @@ -10635,7 +11063,10 @@ def _check_background_cancellation(subtask: str) -> None: # JVNAUTOSCI-1011: Use window session context if available effective = get_effective_context( - request_window_session_id, dict(session), user_concept_id + request_window_session_id, + dict(session), + user_concept_id, + require_known_window=bool(request_window_session_id), ) # Window-session storage uses the organisation slug in some paths. # Canonicalise it at the authenticated request boundary so downstream @@ -10646,6 +11077,21 @@ def _check_background_cancellation(subtask: str) -> None: # Store user_concept_id in session for history tracking if user_concept_id: session["user_concept_id"] = user_concept_id + except WindowSessionContextUnavailable: + return ( + jsonify( + { + "error": "window_context_unavailable", + "detail": ( + "This tab's organisation context must be rebound before " + "the turn can run." + ), + "retryable": True, + "request_id": request_id, + } + ), + 409, + ) except Exception: user_concept_id = None org_concept_id = None @@ -10692,9 +11138,7 @@ def _check_background_cancellation(subtask: str) -> None: ) explicit_resolved_profile = explicit_gmail_authority.get("profile_id") if not explicit_gmail_authority.get("success"): - reason_code = str( - explicit_gmail_authority.get("reason_code") or "" - ) + reason_code = str(explicit_gmail_authority.get("reason_code") or "") unavailable = reason_code in { "authorised_mail_profile_unavailable", "gmail_profile_runtime_configuration_unavailable", @@ -10727,11 +11171,9 @@ def _check_background_cancellation(subtask: str) -> None: gmail_profile_trusted_binding = dict(explicit_trusted_choice) try: - request_workflow_launch_inputs = ( - _authorise_generate_file_copy_launch_input( - request_workflow_launch_inputs, - user_concept_id=user_concept_id, - ) + request_workflow_launch_inputs = _authorise_generate_file_copy_launch_input( + request_workflow_launch_inputs, + user_concept_id=user_concept_id, ) except ValueError as exc: return ( @@ -10759,16 +11201,6 @@ def _check_background_cancellation(subtask: str) -> None: 403, ) - if project_live_progress: - _register_tool_progress_scope_aliases( - request_id=request_id, - primary_scope_key=progress_scope_key, - mirror_scope_keys=progress_mirror_scope_keys, - user_concept_id=user_concept_id, - window_session_id=request_window_session_id, - anonymous_session_id=_progress_str(session.get("tool_progress_scope")), - ) - role_in_org = effective.get("role") if isinstance(effective, dict) else None _emit_context_setup_progress( @@ -10782,6 +11214,20 @@ def _check_background_cancellation(subtask: str) -> None: flask_session_snapshot=dict(session), ) user_namespace = namespace_resolution.get("namespace") + if ( + project_live_progress + and user_concept_id + and user_namespace + and request_window_session_id + and namespace_resolution.get("effective_context_source") == "window_session" + ): + progress_scope_key = _build_authenticated_tool_progress_scope_key( + user_concept_id=user_concept_id, + organisation_concept_id=org_concept_id, + namespace=str(user_namespace), + ) + else: + project_live_progress = False namespace_source = namespace_resolution.get("namespace_source") or "missing" namespace_report: dict[str, object] = { "authenticated": bool(user_concept_id), @@ -10837,6 +11283,35 @@ def _check_background_cancellation(subtask: str) -> None: user_concept_id, ) + if background_mode: + if not user_concept_id or not user_namespace: + return ( + jsonify( + { + "error": "background_task_actor_scope_required", + "detail": ( + "Background turns require an authenticated actor and " + "an exact namespace." + ), + } + ), + 401, + ) + return _submit_generate_background_request( + app=current_app._get_current_object(), + request_data=data if isinstance(data, Mapping) else None, + request_headers=request.headers, + session_snapshot=dict(session), + request_id=request_id, + actor_scope={ + "user_concept_id": user_concept_id, + "organisation_concept_id": org_concept_id, + "namespace": user_namespace, + "role_in_org": role_in_org, + }, + conversation_session_id=request_conversation_session_id, + ) + _emit_context_setup_progress( subtask="conversation session", result_summary="Ensuring a conversation session exists for this replay turn.", @@ -10874,6 +11349,90 @@ def _check_background_cancellation(subtask: str) -> None: ), shared_invite=shared_invite, ) + + # Admission belongs immediately before the canonical history read: two + # turns admitted after that point could observe the same prior transcript + # and append assistants in completion order. Distinct conversation keys + # remain free to run concurrently. + raw_prompt_queue_id = data.get("prompt_queue_id") + prompt_queue_id = ( + raw_prompt_queue_id.strip() + if isinstance(raw_prompt_queue_id, str) and raw_prompt_queue_id.strip() + else None + ) + try: + if user_concept_id and history_user_id and history_namespace: + raw_attempt_id = data.get("attempt_id") + attempt_id = ( + raw_attempt_id.strip() + if isinstance(raw_attempt_id, str) and raw_attempt_id.strip() + else None + ) + raw_session_name = data.get("conversation_session_name") + requested_session_name = ( + raw_session_name.strip() + if isinstance(raw_session_name, str) and raw_session_name.strip() + else created_conversation_session_name + ) + admission_token = conversation_turn_admission_service.acquire( + scope={ + "user_concept_id": user_concept_id, + "organisation_concept_id": org_concept_id, + "namespace": user_namespace, + }, + prompt_raw=prompt_text or "[assistant opening]", + session_id=session_id, + session_name=requested_session_name, + client_request_id=request_id, + conversation_key=build_conversation_key( + owner_user_id=history_user_id, + history_namespace=history_namespace, + conversation_session_id=session_id, + ), + queue_id=prompt_queue_id, + attempt_id=attempt_id, + ) + else: + anonymous_admission_id = session.get("_von_anonymous_admission_id") + if ( + not isinstance(anonymous_admission_id, str) + or not anonymous_admission_id + ): + anonymous_admission_id = str(uuid.uuid4()) + session["_von_anonymous_admission_id"] = anonymous_admission_id + anonymous_fingerprint = hashlib.sha256( + anonymous_admission_id.encode("utf-8") + ).hexdigest() + actor_capacity_key = user_concept_id or f"anonymous:{anonymous_fingerprint}" + fallback_namespace = ( + history_namespace + or user_namespace + or f"anonymous:{anonymous_fingerprint}" + ) + admission_token = conversation_turn_admission_service.acquire_ephemeral( + actor_capacity_key=actor_capacity_key, + conversation_key=build_conversation_key( + owner_user_id=history_user_id or actor_capacity_key, + history_namespace=fallback_namespace, + conversation_session_id=session_id, + ), + client_request_id=request_id, + ) + except ConversationTurnAdmissionError as exc: + response = jsonify( + { + "error": exc.error_code, + "detail": str(exc), + "retryable": True, + "request_id": request_id, + "prompt_queue_id": exc.queue_id or prompt_queue_id, + "retry_after_seconds": 1, + } + ) + response.headers["Retry-After"] = "1" + return response, exc.status_code + setattr(g, _TURN_ADMISSION_CONTEXT_KEY, admission_token) + focal_context_messages: list[dict[str, Any]] = [] focal_context_unavailable: list[str] = [] _conversation_history_meta: dict[str, Any] = {} @@ -10889,9 +11448,7 @@ def _check_background_cancellation(subtask: str) -> None: "retained_count": 0, "total_count": 0, "omitted_count": 0, - "retention_limit": ( - chat_history_service.CONVERSATION_OBSERVATION_MAX_ITEMS - ), + "retention_limit": (chat_history_service.CONVERSATION_OBSERVATION_MAX_ITEMS), } _check_background_cancellation("shared conversation owner") @@ -10929,9 +11486,7 @@ def _check_background_cancellation(subtask: str) -> None: "conversation_observation_state" ) if isinstance(loaded_observation_state, Mapping): - conversation_observation_state = dict( - loaded_observation_state - ) + conversation_observation_state = dict(loaded_observation_state) if ( shared_invite and history_owner_user_id @@ -11047,11 +11602,9 @@ def _check_background_cancellation(subtask: str) -> None: build_gmail_profile_turn_scope, ) - remembered_gmail_scope = ( - latest_resource_scope_from_conversation_situation( - conversation_situation_text, - source_family="gmail", - ) + remembered_gmail_scope = latest_resource_scope_from_conversation_situation( + conversation_situation_text, + source_family="gmail", ) gmail_authority = build_gmail_profile_turn_scope( user_concept_id=user_concept_id, @@ -11490,9 +12043,9 @@ def _check_background_cancellation(subtask: str) -> None: # Update existing system message to include user/org context existing_system = enhanced_context[0]["content"] if not any(part in existing_system for part in system_message_parts): - enhanced_context[0][ - "content" - ] = f"{existing_system} | {' | '.join(system_message_parts)}" + enhanced_context[0]["content"] = ( + f"{existing_system} | {' | '.join(system_message_parts)}" + ) # Represented actor-specific behaviour remains ordinary model context; # it is independent of the retired universal controller. @@ -11874,7 +12427,9 @@ def _emit_stage_progress(info: Mapping[str, Any] | None) -> None: if progress_updates_enabled: def _progress_update(info: dict[str, Any]) -> None: - payload = dict(info) if isinstance(info, dict) else {"status": "unknown"} + payload = ( + dict(info) if isinstance(info, dict) else {"status": "unknown"} + ) payload.setdefault("request_id", request_id) payload.setdefault("goal_label", progress_goal_label) _emit_generate_progress(payload) @@ -12014,8 +12569,7 @@ def _progress_update(info: dict[str, Any]) -> None: ) except Exception as exc: current_app.logger.warning( - "Turn resource presentation planning unavailable for " - "request_id=%s: %s", + "Turn resource presentation planning unavailable for request_id=%s: %s", request_id, exc, ) @@ -12751,9 +13305,7 @@ def _coerce_spoken_text(text: object) -> str | None: ) speech = speech if isinstance(speech, dict) else {} raw_settings = speech.get("settings") - settings = ( - raw_settings if isinstance(raw_settings, dict) else {} - ) + settings = raw_settings if isinstance(raw_settings, dict) else {} preferred = settings.get("preferred_speaking_seconds") maximum = settings.get("max_speaking_seconds") @@ -12821,10 +13373,7 @@ def _coerce_spoken_text(text: object) -> str | None: _record_stage_llm_call( call_type="llm.generate", model_name=model_name, - duration_ms=( - time.perf_counter() - narration_llm_started - ) - * 1000.0, + duration_ms=(time.perf_counter() - narration_llm_started) * 1000.0, usage=None, note="Presenter narration synthesis.", stage="narration", @@ -12887,15 +13436,11 @@ def _coerce_spoken_text(text: object) -> str | None: # Defensive: never fail the request just because narration generation failed. spoken_backfill_error_class = type(exc).__name__ if narration_llm_started is not None: - eligibility_denied = isinstance( - exc, ModelExecutionEligibilityError - ) + eligibility_denied = isinstance(exc, ModelExecutionEligibilityError) _record_stage_llm_call( call_type="llm.generate", model_name=model_name, - duration_ms=( - time.perf_counter() - narration_llm_started - ) + duration_ms=(time.perf_counter() - narration_llm_started) * 1000.0, usage=None, note="Presenter narration synthesis.", @@ -12904,12 +13449,8 @@ def _coerce_spoken_text(text: object) -> str | None: status="failed", success=False, error_class=spoken_backfill_error_class, - failure_kind=( - exc.failure_kind if eligibility_denied else None - ), - provider_request_sent=( - False if eligibility_denied else None - ), + failure_kind=(exc.failure_kind if eligibility_denied else None), + provider_request_sent=(False if eligibility_denied else None), ) spoken_backfill_latency_ms = ( time.perf_counter() - spoken_backfill_started_perf @@ -13003,9 +13544,7 @@ def _coerce_spoken_text(text: object) -> str | None: request_id, exc, ) - if resource_annotation_applied and isinstance( - presenter_channels, Mapping - ): + if resource_annotation_applied and isinstance(presenter_channels, Mapping): presenter_channels = { **dict(presenter_channels), "screen": response_text, @@ -13037,9 +13576,7 @@ def _coerce_spoken_text(text: object) -> str | None: for capsule in presentation_capsules if isinstance(capsule, Mapping) ] - deterministic_situation_projection = ( - projection_with_presentation - ) + deterministic_situation_projection = projection_with_presentation auxiliary_llm_calls.append( { "type": "turn_resource_presentation", @@ -13047,9 +13584,7 @@ def _coerce_spoken_text(text: object) -> str | None: "schema_version" ), "applied": True, - "annotations": resource_presentation_plan.get( - "annotations" - ), + "annotations": resource_presentation_plan.get("annotations"), } ) @@ -13330,9 +13865,7 @@ def _coerce_spoken_text(text: object) -> str | None: "(workflow unavailable)." ), stage="buttonify", - provider=( - exc.provider if eligibility_denied else None - ), + provider=(exc.provider if eligibility_denied else None), status="failed", success=False, error_class=buttonify_error_class, @@ -13426,9 +13959,7 @@ def _coerce_spoken_text(text: object) -> str | None: # for shared conversations. if history_user_id: stored_context_for_stats = [ - dict(message) - for message in context - if isinstance(message, Mapping) + dict(message) for message in context if isinstance(message, Mapping) ] if _user_message_persisted_early: stored_context_for_stats.append( @@ -13438,9 +13969,7 @@ def _coerce_spoken_text(text: object) -> str | None: "author_user_id": user_concept_id, } ) - current_context_stats = _calculate_context_stats( - stored_context_for_stats - ) + current_context_stats = _calculate_context_stats(stored_context_for_stats) else: current_context_stats = _calculate_context_stats( current_app.config["CONTEXT"] @@ -13869,7 +14398,9 @@ def _refresh_llm_debug_timing_payload(debug_payload: dict[str, Any]) -> None: result_body=final_success_body, request_id=request_id, session_id=session_id, - user_id=history_user_id or user_concept_id, + user_id=user_concept_id, + organisation_id=org_concept_id, + namespace=user_namespace, response_text=response_text, ) return jsonify(final_success_body) @@ -14171,9 +14702,7 @@ def history(): ) owner_history = _apply_default_author(owner_history, owner_user_id) invitee_history = _apply_default_author(invitee_history, user_concept_id) - canonical_history = _merge_shared_histories( - owner_history, invitee_history - ) + canonical_history = _merge_shared_histories(owner_history, invitee_history) else: canonical_history = owner_history @@ -14198,9 +14727,7 @@ def history(): history_offset=history_offset, owner_user_id=owner_user_id, ) - segments = chat_history_service._chunk_history_segments( - segments, segment_size - ) + segments = chat_history_service._chunk_history_segments(segments, segment_size) meta = {"history_truncated": history_truncated} total_segments = len(segments) @@ -14742,9 +15269,7 @@ def history_turn_failure_capsule(): return jsonify({"error": "history_index required"}), 400 window_session_id = request.headers.get("X-Von-Window-Session") - effective = get_effective_context( - window_session_id, dict(session), user_concept_id - ) + effective = get_effective_context(window_session_id, dict(session), user_concept_id) actor_namespace, organisation_concept_id = _resolve_history_request_scope_hints( user_concept_id=user_concept_id, effective_context=effective, @@ -14864,9 +15389,7 @@ def history_turn_telemetry_access(): requested_history_index = request.args.get("history_index", type=int) window_session_id = request.headers.get("X-Von-Window-Session") - effective = get_effective_context( - window_session_id, dict(session), user_concept_id - ) + effective = get_effective_context(window_session_id, dict(session), user_concept_id) actor_namespace, organisation_concept_id = _resolve_history_request_scope_hints( user_concept_id=user_concept_id, effective_context=effective, @@ -14902,12 +15425,9 @@ def history_turn_telemetry_access(): else None ) organisation_concept_id = shared_org_id or organisation_concept_id - owner_namespace = ( - _derive_namespace_for_user_org( - owner_user_id, organisation_concept_id - ) - or chat_history_service.resolve_chat_history_namespace(owner_user_id) - ) + owner_namespace = _derive_namespace_for_user_org( + owner_user_id, organisation_concept_id + ) or chat_history_service.resolve_chat_history_namespace(owner_user_id) diagnostics: Mapping[str, Any] | None = None if requested_session_id and requested_history_index is not None: @@ -14932,7 +15452,9 @@ def history_turn_telemetry_access(): else None ) if compact_request_id and compact_request_id != request_id: - return jsonify({"error": "request_id does not belong to history_index"}), 403 + return jsonify( + {"error": "request_id does not belong to history_index"} + ), 403 if compact_request_id == request_id: # The authenticated exact history location is sufficient to issue a # read-only descriptor. Avoid hydrating the full diagnostics body @@ -15029,9 +15551,7 @@ def history_turn_telemetry_access(): owner_user_id = diagnostics_owner_user_id if isinstance(resolved_invite, Mapping): organisation_concept_id = ( - _normalise_concept_id( - resolved_invite.get("organisation_concept_id") - ) + _normalise_concept_id(resolved_invite.get("organisation_concept_id")) or organisation_concept_id ) elif diagnostics_owner_user_id: @@ -16159,21 +16679,17 @@ def reset_context(): ), shared_invite=shared_invite, ) - reset_outcome = ( - chat_history_service.reset_chat_history_conversation_state( - user_id=owner_user_id, - session_id=session_id, - updated_by=user_concept_id, - namespace=owner_namespace, - ) + reset_outcome = chat_history_service.reset_chat_history_conversation_state( + user_id=owner_user_id, + session_id=session_id, + updated_by=user_concept_id, + namespace=owner_namespace, ) if not bool( isinstance(reset_outcome, Mapping) and reset_outcome.get("matched") is True ): - raise RuntimeError( - "Conversation session was not found during reset." - ) + raise RuntimeError("Conversation session was not found during reset.") # Clear the conversation context current_app.config["CONTEXT"] = [] @@ -17493,8 +18009,7 @@ def search_conversations(): cursor=request.args.get("cursor"), include_hidden=request.args.get("include_hidden", "false").lower() == "true", - trashed_only=request.args.get("trashed_only", "false").lower() - == "true", + trashed_only=request.args.get("trashed_only", "false").lower() == "true", ) return jsonify(result) except WindowSessionOwnershipError: @@ -17508,7 +18023,9 @@ def search_conversations(): 403, ) except conversation_search_service.ConversationSearchError as exc: - return jsonify({"error": str(exc), "error_code": "conversation_search_failed"}), 400 + return jsonify( + {"error": str(exc), "error_code": "conversation_search_failed"} + ), 400 except Exception: current_app.logger.exception("Unexpected error searching conversations") return ( @@ -17532,10 +18049,7 @@ def delete_chat_session(): ) user_concept_id, actor_source = get_effective_user_concept_id_with_source() - if ( - not user_concept_id - or actor_source == LEGACY_IDENTITY_HEADER_ACTOR_SOURCE - ): + if not user_concept_id or actor_source == LEGACY_IDENTITY_HEADER_ACTOR_SOURCE: return jsonify({"error": "Not authenticated"}), 401 data = request.get_json(silent=True) or {} @@ -17860,9 +18374,12 @@ def chat_session_focus(): if not isinstance(session_id, str) or not session_id.strip(): return jsonify({"error": "session_id required"}), 400 session_id = session_id.strip() - if get_accepted_invite_for_user_session( - user_concept_id=user_concept_id, session_id=session_id - ) is not None: + if ( + get_accepted_invite_for_user_session( + user_concept_id=user_concept_id, session_id=session_id + ) + is not None + ): return ( jsonify({"error": "Only the conversation owner may change focus"}), 403, @@ -18657,10 +19174,7 @@ def get_my_organisations(): from ...services.organisation_membership_service import get_user_memberships user_concept_id, actor_source = get_effective_user_concept_id_with_source() - if ( - not user_concept_id - or actor_source == LEGACY_IDENTITY_HEADER_ACTOR_SOURCE - ): + if not user_concept_id or actor_source == LEGACY_IDENTITY_HEADER_ACTOR_SOURCE: return jsonify({"error": "Not authenticated"}), 401 def _prettify_concept_id(concept_id: str) -> str: diff --git a/src/backend/services/background_task_service.py b/src/backend/services/background_task_service.py index 71b5a3edf..8d3baf3e0 100644 --- a/src/backend/services/background_task_service.py +++ b/src/backend/services/background_task_service.py @@ -42,11 +42,21 @@ # Maximum concurrent background tasks _MAX_WORKERS = 4 +_MAX_PENDING_TASKS = 32 +_MAX_ACTIVE_TASKS_PER_USER = 2 _TERMINAL_STATUSES = {"completed", "failed", "cancelled"} _PROGRESS_HISTORY_LIMIT = 200 +class BackgroundTaskScopeMismatch(ValueError): + """A terminal observation conflicts with the task's bound actor scope.""" + + +class BackgroundTaskCapacityReached(RuntimeError): + """Bounded background admission is full for this server or actor.""" + + def _is_active_status(status: str) -> bool: return status in {"pending", "running"} @@ -84,6 +94,11 @@ class TaskStatus: # For result retrieval filtering (Phase 4) session_id: str | None = None user_id: str | None = None + organisation_id: str | None = None + namespace: str | None = None + queue_id: str | None = None + client_request_id: str | None = None + attempt_id: str | None = None def to_dict(self) -> dict[str, Any]: """Serialise status to a dict suitable for JSON response.""" @@ -102,6 +117,11 @@ def to_dict(self) -> dict[str, Any]: "cancellation_requested": self.cancellation_requested, "session_id": self.session_id, "user_id": self.user_id, + "organisation_id": self.organisation_id, + "namespace": self.namespace, + "queue_id": self.queue_id, + "client_request_id": self.client_request_id, + "attempt_id": self.attempt_id, } @@ -117,6 +137,8 @@ def __init__( *, max_workers: int = _MAX_WORKERS, result_ttl_sec: float = _DEFAULT_RESULT_TTL_SEC, + max_pending_tasks: int = _MAX_PENDING_TASKS, + max_active_tasks_per_user: int = _MAX_ACTIVE_TASKS_PER_USER, ) -> None: self._lock = threading.Lock() self._tasks: dict[str, TaskStatus] = {} @@ -124,6 +146,12 @@ def __init__( self._executor = ThreadPoolExecutor( max_workers=max_workers, thread_name_prefix="bg_task_" ) + self._max_workers = max(1, int(max_workers)) + self._max_pending_tasks = max(0, int(max_pending_tasks)) + self._max_active_tasks_per_user = max( + 1, + min(int(max_active_tasks_per_user), self._max_workers), + ) self._result_ttl_sec = result_ttl_sec self._cleanup_interval_sec = 60.0 self._last_cleanup = time.monotonic() @@ -138,6 +166,11 @@ def submit_task( progress_callback: Callable[[Mapping[str, Any]], None] | None = None, session_id: str | None = None, user_id: str | None = None, + organisation_id: str | None = None, + namespace: str | None = None, + queue_id: str | None = None, + client_request_id: str | None = None, + attempt_id: str | None = None, ) -> TaskStatus: """Submit a callable to run in the background. @@ -149,6 +182,11 @@ def submit_task( progress_callback: Optional callback for progress updates. session_id: Optional session ID for filtering (Phase 4). user_id: Optional user ID for filtering (Phase 4). + organisation_id: Optional organisation ID for actor-scope filtering. + namespace: Optional namespace for actor-scope filtering. + queue_id: Optional persisted prompt queue correlation. + client_request_id: Optional browser request correlation. + attempt_id: Optional exact prompt execution attempt. Returns: Initial TaskStatus with status="pending". @@ -162,17 +200,39 @@ def submit_task( with self._lock: existing = self._tasks.get(task_id) - if existing and existing.status in ("pending", "running"): + if existing: + # Client request IDs are correlation, never overwrite authority. + # Keep every task ID single-use until normal TTL cleanup removes + # the terminal record; a replay can then receive a fresh task ID. raise ValueError( f"Task {task_id} already exists and is {existing.status}" ) + active_tasks = [ + task for task in self._tasks.values() if _is_active_status(task.status) + ] + if len(active_tasks) >= self._max_workers + self._max_pending_tasks: + raise BackgroundTaskCapacityReached( + "Von is at its bounded background-task capacity" + ) + if user_id is not None: + user_active = sum(task.user_id == user_id for task in active_tasks) + if user_active >= self._max_active_tasks_per_user: + raise BackgroundTaskCapacityReached( + "This user is at the background-task capacity" + ) + status = TaskStatus( task_id=task_id, status="pending", created_at=now, session_id=session_id, user_id=user_id, + organisation_id=organisation_id, + namespace=namespace, + queue_id=queue_id, + client_request_id=client_request_id, + attempt_id=attempt_id, ) self._tasks[task_id] = status @@ -180,7 +240,11 @@ def _run_task() -> Any: # Mark as running with self._lock: task_status = self._tasks.get(task_id) - if task_status and _is_active_status(task_status.status): + if ( + task_status + and _is_active_status(task_status.status) + and not task_status.cancellation_requested + ): task_status.status = "running" task_status.started_at = datetime.now(timezone.utc) @@ -199,11 +263,7 @@ def _update_progress(info: Mapping[str, Any]) -> None: try: with self._lock: task_status = self._tasks.get(task_id) - if ( - task_status - and task_status.status == "cancelled" - and task_status.cancellation_requested - ): + if task_status and task_status.cancellation_requested: raise CancellationRequested(task_id=task_id) # Inject progress callback if supported @@ -231,26 +291,49 @@ def _update_progress(info: Mapping[str, Any]) -> None: with self._lock: task_status = self._tasks.get(task_id) if task_status and _is_active_status(task_status.status): - task_status.status = "cancelled" - task_status.completed_at = datetime.now(timezone.utc) - task_status.error = str(exc) - task_status.progress["status"] = "cancelled" - _append_progress_history(task_status, task_status.progress) + self._terminalise_cancelled_locked( + task_id, + task_status, + error=str(exc), + ) raise except Exception as exc: - _logger.exception( - "[background_task] Task %s failed: %s", task_id, exc - ) + cancellation_won = False with self._lock: task_status = self._tasks.get(task_id) if task_status and _is_active_status(task_status.status): - task_status.status = "failed" - task_status.completed_at = datetime.now(timezone.utc) - task_status.error = str(exc) - task_status.progress["status"] = "failed" - task_status.progress["error"] = str(exc) - _append_progress_history(task_status, task_status.progress) + if task_status.cancellation_requested: + # Cancellation can win immediately before a + # queue/admission operation raises. The work did + # not complete, so acknowledge the user's prior + # cancellation instead of publishing a spurious + # failed task. + self._terminalise_cancelled_locked( + task_id, + task_status, + error=str(exc), + ) + cancellation_won = True + else: + task_status.status = "failed" + task_status.completed_at = datetime.now(timezone.utc) + task_status.error = str(exc) + task_status.progress["status"] = "failed" + task_status.progress["error"] = str(exc) + _append_progress_history( + task_status, + task_status.progress, + ) + if cancellation_won: + _logger.info( + "[background_task] Task %s acknowledged cancellation " + "after its callable stopped with %s", + task_id, + exc, + ) + else: + _logger.exception("[background_task] Task %s failed", task_id) raise future = self._executor.submit(_run_task) @@ -269,6 +352,8 @@ def mark_terminal_external( progress: Mapping[str, Any] | None = None, session_id: str | None = None, user_id: str | None = None, + organisation_id: str | None = None, + namespace: str | None = None, ) -> TaskStatus: """Record an externally observed terminal task state. @@ -299,11 +384,29 @@ def mark_terminal_external( progress=progress_payload, session_id=session_id, user_id=user_id, + organisation_id=organisation_id, + namespace=namespace, ) _append_progress_history(task_status, progress_payload) self._tasks[task_id] = task_status return task_status + incoming_scope = { + "user_id": user_id, + "organisation_id": organisation_id, + "namespace": namespace, + } + for field_name, incoming_value in incoming_scope.items(): + bound_value = getattr(task_status, field_name) + if ( + bound_value is not None + and incoming_value is not None + and bound_value != incoming_value + ): + raise BackgroundTaskScopeMismatch( + "Terminal task observation does not match the bound actor scope" + ) + if task_status.status in _TERMINAL_STATUSES: # The first terminal observation is final. A worker, durable # poller, or cancellation path may arrive later, but it cannot @@ -322,6 +425,10 @@ def mark_terminal_external( task_status.session_id = session_id if task_status.user_id is None and user_id is not None: task_status.user_id = user_id + if task_status.organisation_id is None and organisation_id is not None: + task_status.organisation_id = organisation_id + if task_status.namespace is None and namespace is not None: + task_status.namespace = namespace return task_status def get_task_status(self, task_id: str) -> TaskStatus | None: @@ -336,6 +443,42 @@ def get_task_status(self, task_id: str) -> TaskStatus | None: with self._lock: return self._tasks.get(task_id) + @staticmethod + def _status_matches_scope( + status: TaskStatus, + *, + user_id: str, + organisation_id: str | None, + namespace: str | None, + ) -> bool: + return bool( + status.user_id == user_id + and status.organisation_id == organisation_id + and status.namespace == namespace + ) + + def get_task_status_for_scope( + self, + task_id: str, + *, + user_id: str, + organisation_id: str | None, + namespace: str | None, + ) -> TaskStatus | None: + """Return a task only inside its immutable actor/organisation scope.""" + + self._maybe_cleanup() + with self._lock: + status = self._tasks.get(task_id) + if status is None or not self._status_matches_scope( + status, + user_id=user_id, + organisation_id=organisation_id, + namespace=namespace, + ): + return None + return status + def update_progress(self, task_id: str, progress: Mapping[str, Any]) -> bool: """Update progress for an active task. @@ -393,32 +536,87 @@ def request_cancellation(self, task_id: str) -> bool: """ with self._lock: status = self._tasks.get(task_id) - if status is None: - return False - if status.status not in ("pending", "running"): - return False - status.cancellation_requested = True - status.status = "cancelled" - if status.started_at is None: - status.started_at = datetime.now(timezone.utc) - status.completed_at = datetime.now(timezone.utc) - status.error = f"Cancellation requested for task {task_id}" - progress_payload = dict(status.progress) - progress_payload.update( - { - "status": "cancelled", - "phase": "cancelled", - "phase_label": "Cancelled", - "result_summary": ( - "Cancellation requested for the background generate task." - ), - } + return self._request_cancellation_locked(task_id, status) + + def _request_cancellation_locked( + self, + task_id: str, + status: TaskStatus | None, + ) -> bool: + if status is None or status.status not in ("pending", "running"): + return False + status.cancellation_requested = True + future = self._futures.get(task_id) + if status.status == "pending" and future is not None and future.cancel(): + self._terminalise_cancelled_locked( + task_id, + status, + error=f"Cancellation acknowledged before task {task_id} started", ) - status.progress = progress_payload - _append_progress_history(status, progress_payload) - _logger.info("[background_task] Task %s marked cancelled", task_id) return True + progress_payload = dict(status.progress) + progress_payload.update( + { + "status": "cancelling", + "phase": "cancelling", + "phase_label": "Cancelling", + "result_summary": ( + "Cancellation requested; waiting for the running task to stop." + ), + } + ) + status.progress = progress_payload + _append_progress_history(status, progress_payload) + _logger.info("[background_task] Cancellation requested for task %s", task_id) + return True + + @staticmethod + def _terminalise_cancelled_locked( + task_id: str, + status: TaskStatus, + *, + error: str, + ) -> None: + status.status = "cancelled" + if status.started_at is None: + status.started_at = datetime.now(timezone.utc) + status.completed_at = datetime.now(timezone.utc) + status.error = error + progress_payload = dict(status.progress) + progress_payload.update( + { + "status": "cancelled", + "phase": "cancelled", + "phase_label": "Cancelled", + "result_summary": "The background task stopped after cancellation.", + } + ) + status.progress = progress_payload + _append_progress_history(status, progress_payload) + _logger.info("[background_task] Task %s acknowledged cancellation", task_id) + + def request_cancellation_for_scope( + self, + task_id: str, + *, + user_id: str, + organisation_id: str | None, + namespace: str | None, + ) -> bool: + """Request cancellation only for the exact bound actor scope.""" + + with self._lock: + status = self._tasks.get(task_id) + if status is None or not self._status_matches_scope( + status, + user_id=user_id, + organisation_id=organisation_id, + namespace=namespace, + ): + return False + return self._request_cancellation_locked(task_id, status) + def is_cancellation_requested(self, task_id: str) -> bool: """Check if cancellation was requested for a task. @@ -456,13 +654,17 @@ def list_tasks( status_filter: str | None = None, session_id: str | None = None, user_id: str | None = None, + organisation_id: str | None = None, + namespace: str | None = None, ) -> list[dict[str, Any]]: - """List all tasks, optionally filtered by status, session, or user. + """List all tasks, optionally filtered by status or actor scope. Args: status_filter: If provided, only return tasks with this status. session_id: If provided, only return tasks for this session (Phase 4). user_id: If provided, only return tasks for this user (Phase 4). + organisation_id: If provided, only return tasks for this organisation. + namespace: If provided, only return tasks for this namespace. Returns: List of task status dicts. @@ -476,9 +678,42 @@ def list_tasks( tasks = [t for t in tasks if t.session_id == session_id] if user_id: tasks = [t for t in tasks if t.user_id == user_id] + if organisation_id: + tasks = [t for t in tasks if t.organisation_id == organisation_id] + if namespace: + tasks = [t for t in tasks if t.namespace == namespace] return [t.to_dict() for t in tasks] + def list_tasks_for_scope( + self, + *, + user_id: str, + organisation_id: str | None, + namespace: str | None, + status_filter: str | None = None, + session_id: str | None = None, + ) -> list[dict[str, Any]]: + """List tasks only when every bound actor-scope component matches.""" + + self._maybe_cleanup() + with self._lock: + tasks = [ + task + for task in self._tasks.values() + if self._status_matches_scope( + task, + user_id=user_id, + organisation_id=organisation_id, + namespace=namespace, + ) + ] + if status_filter: + tasks = [task for task in tasks if task.status == status_filter] + if session_id: + tasks = [task for task in tasks if task.session_id == session_id] + return [task.to_dict() for task in tasks] + def _maybe_cleanup(self) -> None: """Clean up old completed/failed tasks if cleanup interval has passed.""" now = time.monotonic() diff --git a/src/backend/services/chat_prompt_queue_service.py b/src/backend/services/chat_prompt_queue_service.py index e47e45527..fde4a4648 100644 --- a/src/backend/services/chat_prompt_queue_service.py +++ b/src/backend/services/chat_prompt_queue_service.py @@ -6,12 +6,16 @@ from __future__ import annotations +import hashlib +import os +import threading from datetime import datetime, timedelta, timezone from typing import Any, Mapping, Sequence from uuid import uuid4 from pymongo import ReturnDocument from pymongo.collection import Collection +from pymongo.errors import DuplicateKeyError from ..db.mongo_client import get_chat_prompt_queue_collection from .namespace_service import ( @@ -72,11 +76,52 @@ def __init__( self.details = dict(details or {}) +class ConversationTurnAlreadyActive(ChatPromptQueueError): + """Raised when another admitted turn owns the conversation carrier.""" + + def __init__(self, message: str, *, queue_id: str | None = None) -> None: + super().__init__(message) + self.queue_id = queue_id + + +class ChatPromptQueueCapacityReached(ChatPromptQueueError): + """Raised when the bounded foreground backlog has no free slot.""" + + def __init__(self, message: str, *, limit_kind: str) -> None: + super().__init__(message) + self.limit_kind = limit_kind + + +_QUEUE_ADMISSION_LOCK = threading.RLock() +_UNSET = object() + + +def _positive_int_env(name: str, default: int) -> int: + try: + return max(1, int(os.getenv(name, str(default)))) + except (TypeError, ValueError): + return default + + +def _foreground_queue_limits() -> tuple[int, int]: + return ( + _positive_int_env("VON_MAX_QUEUED_TURNS_GLOBAL", 500), + _positive_int_env("VON_MAX_QUEUED_TURNS_PER_USER", 25), + ) + + +def _queued_user_slot_key(user_concept_id: str, slot: int) -> str: + actor_hash = hashlib.sha256(user_concept_id.encode("utf-8")).hexdigest() + return f"{actor_hash}:{slot}" + + def _now() -> datetime: return datetime.now(timezone.utc) -def _coerce_text(value: Any, *, field: str, required: bool, max_chars: int | None = None) -> str | None: +def _coerce_text( + value: Any, *, field: str, required: bool, max_chars: int | None = None +) -> str | None: if value is None: if required: raise InvalidChatPromptQueueInput(f"{field} is required") @@ -340,6 +385,12 @@ def serialise_queue_record(doc: Mapping[str, Any] | None) -> dict[str, Any] | No "completed_at": _serialise_datetime(doc.get("completed_at")), "last_error": doc.get("last_error"), "source": doc.get("source"), + "client_request_id": doc.get("client_request_id"), + "attempt_id": doc.get("attempt_id"), + "conversation_key": doc.get("conversation_key"), + "server_instance_id": doc.get("server_instance_id"), + "lease_acquired_at": _serialise_datetime(doc.get("lease_acquired_at")), + "lease_heartbeat_at": _serialise_datetime(doc.get("lease_heartbeat_at")), "stale_advisory": bool(doc.get("stale_advisory")), "stale_advisory_at": _serialise_datetime(doc.get("stale_advisory_at")), "stale_advisory_reason": doc.get("stale_advisory_reason"), @@ -353,7 +404,9 @@ def list_active_queue_records( statuses: Sequence[str] = ACTIVE_STATUSES, limit: int = 100, ) -> list[dict[str, Any]]: - allowed_statuses = [str(status) for status in statuses if str(status) in VALID_STATUSES] + allowed_statuses = [ + str(status) for status in statuses if str(status) in VALID_STATUSES + ] if not allowed_statuses: allowed_statuses = list(ACTIVE_STATUSES) max_limit = max(1, min(int(limit or 100), 500)) @@ -364,7 +417,9 @@ def list_active_queue_records( "status": {"$in": allowed_statuses}, } docs = _collection().find(query).sort([("created_at", 1)]).limit(max_limit) - return [record for record in (serialise_queue_record(doc) for doc in docs) if record] + return [ + record for record in (serialise_queue_record(doc) for doc in docs) if record + ] def list_recent_failed_queue_records( @@ -389,10 +444,14 @@ def list_recent_failed_queue_records( .sort([("completed_at", -1), ("updated_at", -1)]) .limit(max_limit) ) - return [record for record in (serialise_queue_record(doc) for doc in docs) if record] + return [ + record for record in (serialise_queue_record(doc) for doc in docs) if record + ] -def list_queue_visibility_records(*, scope: Mapping[str, Any]) -> dict[str, list[dict[str, Any]]]: +def list_queue_visibility_records( + *, scope: Mapping[str, Any] +) -> dict[str, list[dict[str, Any]]]: """Return active queue records plus bounded failed records for UI visibility.""" return { @@ -409,6 +468,9 @@ def create_queue_record( session_name: Any = None, status: str = STATUS_QUEUED, source: str = "queued", + client_request_id: Any = None, + attempt_id: Any = None, + conversation_key: Any = None, ) -> dict[str, Any]: if status not in {STATUS_QUEUED, STATUS_IN_PROGRESS}: raise InvalidChatPromptQueueInput("status must be queued or in_progress") @@ -418,7 +480,9 @@ def create_queue_record( required=True, max_chars=MAX_PROMPT_RAW_CHARS, ) - session_id_clean = _coerce_scope_value(session_id, field="session_id", required=False) + session_id_clean = _coerce_scope_value( + session_id, field="session_id", required=False + ) session_name_clean = _coerce_text( session_name, field="session_name", @@ -427,6 +491,21 @@ def create_queue_record( ) if session_name_clean is not None: session_name_clean = session_name_clean.strip() or None + client_request_id_clean = _coerce_scope_value( + client_request_id, + field="client_request_id", + required=False, + ) + attempt_id_clean = _coerce_scope_value( + attempt_id, + field="attempt_id", + required=False, + ) + conversation_key_clean = _coerce_scope_value( + conversation_key, + field="conversation_key", + required=False, + ) now = _now() doc = { "queue_id": str(uuid4()), @@ -436,6 +515,9 @@ def create_queue_record( "session_id": session_id_clean, "session_name": session_name_clean, "source": source, + "client_request_id": client_request_id_clean, + "attempt_id": attempt_id_clean, + "conversation_key": conversation_key_clean, "attempt_count": 1 if status == STATUS_IN_PROGRESS else 0, "created_at": now, "updated_at": now, @@ -444,20 +526,366 @@ def create_queue_record( "completed_at": None, "last_error": None, } - _collection().insert_one(doc) + coll = _collection() + if status == STATUS_QUEUED: + global_limit, per_user_limit = _foreground_queue_limits() + user_id = str(doc["user_concept_id"]) + with _QUEUE_ADMISSION_LOCK: + legacy_global = coll.count_documents( + { + "status": STATUS_QUEUED, + "queued_global_slot": {"$exists": False}, + } + ) + legacy_user = coll.count_documents( + { + "status": STATUS_QUEUED, + "user_concept_id": user_id, + "queued_user_slot": {"$exists": False}, + } + ) + available_global = max(0, global_limit - int(legacy_global)) + available_user = max(0, per_user_limit - int(legacy_user)) + occupied_global_slots = set( + coll.distinct("queued_global_slot", {"status": STATUS_QUEUED}) + ) + occupied_user_slots = set( + coll.distinct( + "queued_user_slot", + {"status": STATUS_QUEUED, "user_concept_id": user_id}, + ) + ) + inserted = False + for user_slot in range(available_user): + user_slot_key = _queued_user_slot_key(user_id, user_slot) + if user_slot_key in occupied_user_slots: + continue + for global_slot in range(available_global): + if global_slot in occupied_global_slots: + continue + candidate = { + **doc, + "queued_global_slot": global_slot, + "queued_user_slot": user_slot_key, + } + try: + coll.insert_one(candidate) + except DuplicateKeyError: + continue + doc = candidate + inserted = True + break + if inserted: + break + if not inserted: + user_queued = coll.count_documents( + {"status": STATUS_QUEUED, "user_concept_id": user_id} + ) + if user_queued >= per_user_limit: + raise ChatPromptQueueCapacityReached( + "This user has reached the queued-turn backlog limit", + limit_kind="user", + ) + raise ChatPromptQueueCapacityReached( + "Von has reached the global queued-turn backlog limit", + limit_kind="global", + ) + else: + coll.insert_one(doc) record = serialise_queue_record(doc) if record is None: # pragma: no cover - defensive raise ChatPromptQueueUnavailable("created queue record could not be serialised") return record +def _requeue_with_bounded_slots( + *, + coll: Collection, + query: Mapping[str, Any], + update: Mapping[str, Any], + user_concept_id: str, +) -> Mapping[str, Any] | None: + """Atomically re-admit one existing record to the bounded queued backlog.""" + + global_limit, per_user_limit = _foreground_queue_limits() + with _QUEUE_ADMISSION_LOCK: + legacy_global = coll.count_documents( + {"status": STATUS_QUEUED, "queued_global_slot": {"$exists": False}} + ) + legacy_user = coll.count_documents( + { + "status": STATUS_QUEUED, + "user_concept_id": user_concept_id, + "queued_user_slot": {"$exists": False}, + } + ) + available_global = max(0, global_limit - int(legacy_global)) + available_user = max(0, per_user_limit - int(legacy_user)) + occupied_global_slots = set( + coll.distinct("queued_global_slot", {"status": STATUS_QUEUED}) + ) + occupied_user_slots = set( + coll.distinct( + "queued_user_slot", + {"status": STATUS_QUEUED, "user_concept_id": user_concept_id}, + ) + ) + for user_slot in range(available_user): + user_slot_key = _queued_user_slot_key(user_concept_id, user_slot) + if user_slot_key in occupied_user_slots: + continue + for global_slot in range(available_global): + if global_slot in occupied_global_slots: + continue + candidate_update = { + key: dict(value) if isinstance(value, Mapping) else value + for key, value in update.items() + } + set_fields = dict(candidate_update.get("$set") or {}) + set_fields.update( + { + "queued_global_slot": global_slot, + "queued_user_slot": user_slot_key, + } + ) + candidate_update["$set"] = set_fields + try: + doc = coll.find_one_and_update( + dict(query), + candidate_update, + return_document=ReturnDocument.AFTER, + ) + except DuplicateKeyError: + continue + if doc is not None: + return doc + return None + + user_queued = coll.count_documents( + {"status": STATUS_QUEUED, "user_concept_id": user_concept_id} + ) + if user_queued >= per_user_limit: + raise ChatPromptQueueCapacityReached( + "This user has reached the queued-turn backlog limit", + limit_kind="user", + ) + raise ChatPromptQueueCapacityReached( + "Von has reached the global queued-turn backlog limit", + limit_kind="global", + ) + + +def bind_queue_record_to_turn( + *, + scope: Mapping[str, Any], + queue_id: str, + client_request_id: str, + conversation_key: str, + server_instance_id: str, + attempt_id: str | None = None, +) -> dict[str, Any]: + """Atomically bind one queued prompt to the executing conversation turn. + + ``active_conversation_key`` exists only for the admitted record. Its unique + partial index is the cross-thread/process single-flight fence; the stable + ``conversation_key`` remains afterwards for reconciliation. + """ + + queue_id_clean = _coerce_scope_value(queue_id, field="queue_id") + request_id_clean = _coerce_scope_value( + client_request_id, + field="client_request_id", + ) + conversation_key_clean = _coerce_scope_value( + conversation_key, + field="conversation_key", + ) + server_instance_id_clean = _coerce_scope_value( + server_instance_id, + field="server_instance_id", + ) + attempt_id_clean = _coerce_scope_value( + attempt_id, + field="attempt_id", + required=False, + ) or str(uuid4()) + now = _now() + canonical_scope = _scope_query(scope) + coll = _collection() + + existing = coll.find_one( + {**_compatible_scope_query(scope), "queue_id": queue_id_clean} + ) + if existing is None: + raise _transition_not_found_error( + scope=scope, + queue_id=queue_id_clean, + expected_statuses=ACTIVE_STATUSES, + fallback_message="active prompt was not found", + ) + existing_request_id = existing.get("client_request_id") + if existing_request_id and existing_request_id != request_id_clean: + raise InvalidChatPromptQueueInput( + "prompt queue record is already bound to a different request" + ) + if existing.get("active_conversation_key"): + raise ConversationTurnAlreadyActive( + "prompt queue record is already executing", + queue_id=queue_id_clean, + ) + + persisted_conversation_key = _coerce_scope_value( + existing.get("conversation_key"), + field="conversation_key", + required=False, + ) + if ( + persisted_conversation_key + and persisted_conversation_key != conversation_key_clean + ): + raise InvalidChatPromptQueueInput( + "prompt queue record belongs to a different conversation" + ) + + # New records carry the server-computed canonical key from creation, so an + # invitee and owner share one FIFO even though their visibility scopes differ. + # Legacy rows retain their former same-scope session FIFO until first bound. + session_id = existing.get("session_id") + if persisted_conversation_key: + created_at = existing.get("created_at", now) + earlier = coll.find_one( + { + "conversation_key": persisted_conversation_key, + "status": {"$in": ACTIVE_STATUSES}, + "queue_id": {"$ne": queue_id_clean}, + "$or": [ + {"created_at": {"$lt": created_at}}, + { + "created_at": created_at, + "queue_id": {"$lt": queue_id_clean}, + }, + ], + }, + { + "queue_id": 1, + "user_concept_id": 1, + "organisation_concept_id": 1, + "namespace": 1, + }, + sort=[("created_at", 1), ("queue_id", 1)], + ) + if earlier is not None: + same_scope_queue_id = ( + str(earlier.get("queue_id") or "") or None + if _scope_matches(earlier, scope) + else None + ) + raise ConversationTurnAlreadyActive( + "an earlier prompt is waiting for this conversation", + queue_id=same_scope_queue_id, + ) + elif session_id: + earlier = coll.find_one( + { + **_compatible_scope_query(scope), + "session_id": session_id, + "status": {"$in": ACTIVE_STATUSES}, + "queue_id": {"$ne": queue_id_clean}, + "created_at": {"$lt": existing.get("created_at", now)}, + }, + {"queue_id": 1}, + sort=[("created_at", 1), ("queue_id", 1)], + ) + if earlier is not None: + raise ConversationTurnAlreadyActive( + "an earlier prompt is waiting for this conversation", + queue_id=str(earlier.get("queue_id") or "") or None, + ) + + try: + doc = coll.find_one_and_update( + { + **_compatible_scope_query(scope), + "queue_id": queue_id_clean, + "status": {"$in": ACTIVE_STATUSES}, + "$or": [ + {"client_request_id": None}, + {"client_request_id": request_id_clean}, + {"client_request_id": {"$exists": False}}, + ], + "active_conversation_key": {"$exists": False}, + }, + { + "$set": { + **canonical_scope, + "status": STATUS_IN_PROGRESS, + "client_request_id": request_id_clean, + "attempt_id": attempt_id_clean, + "conversation_key": conversation_key_clean, + "active_conversation_key": conversation_key_clean, + "server_instance_id": server_instance_id_clean, + "claimed_at": now, + "lease_acquired_at": now, + "lease_heartbeat_at": now, + "updated_at": now, + "completed_at": None, + "last_error": None, + }, + "$unset": { + "queued_global_slot": "", + "queued_user_slot": "", + "stale_advisory": "", + "stale_advisory_at": "", + "stale_advisory_reason": "", + "reconciliation_required": "", + }, + "$inc": {"attempt_count": 1}, + }, + return_document=ReturnDocument.AFTER, + ) + except DuplicateKeyError as exc: + raise ConversationTurnAlreadyActive( + "another turn is active for this conversation" + ) from exc + + record = serialise_queue_record(doc) + if record is None: + raise ConversationTurnAlreadyActive( + "another turn acquired this prompt or conversation" + ) + return record + + +def heartbeat_bound_turn( + *, + scope: Mapping[str, Any], + queue_id: str, + attempt_id: str, +) -> bool: + """Refresh an exact admitted turn lease without changing its semantics.""" + + now = _now() + result = _collection().update_one( + { + **_compatible_scope_query(scope), + "queue_id": _coerce_scope_value(queue_id, field="queue_id"), + "attempt_id": _coerce_scope_value(attempt_id, field="attempt_id"), + "status": STATUS_IN_PROGRESS, + "active_conversation_key": {"$exists": True}, + }, + {"$set": {"lease_heartbeat_at": now, "updated_at": now}}, + ) + return bool(getattr(result, "modified_count", 0)) + + def update_queued_record( *, scope: Mapping[str, Any], queue_id: str, prompt_raw: str | None = None, - session_id: str | None = None, + session_id: Any = _UNSET, session_name: str | None = None, + conversation_key: Any = _UNSET, ) -> dict[str, Any]: queue_id_clean = _coerce_scope_value(queue_id, field="queue_id") set_fields: dict[str, Any] = {"updated_at": _now()} @@ -468,8 +896,15 @@ def update_queued_record( required=True, max_chars=MAX_PROMPT_RAW_CHARS, ) - if session_id is not None: - set_fields["session_id"] = _coerce_scope_value(session_id, field="session_id", required=False) + if session_id is not _UNSET: + set_fields["session_id"] = _coerce_scope_value( + session_id, field="session_id", required=False + ) + set_fields["conversation_key"] = _coerce_scope_value( + None if conversation_key is _UNSET else conversation_key, + field="conversation_key", + required=False, + ) if session_name is not None: clean_session_name = _coerce_text( session_name, @@ -477,7 +912,9 @@ def update_queued_record( required=False, max_chars=MAX_SESSION_NAME_CHARS, ) - set_fields["session_name"] = clean_session_name.strip() if clean_session_name else None + set_fields["session_name"] = ( + clean_session_name.strip() if clean_session_name else None + ) canonical_scope = _scope_query(scope) doc = _collection().find_one_and_update( @@ -520,6 +957,8 @@ def claim_queue_record(*, scope: Mapping[str, Any], queue_id: str) -> dict[str, "last_error": None, }, "$unset": { + "queued_global_slot": "", + "queued_user_slot": "", "stale_advisory": "", "stale_advisory_at": "", "stale_advisory_reason": "", @@ -540,35 +979,104 @@ def claim_queue_record(*, scope: Mapping[str, Any], queue_id: str) -> dict[str, return record -def requeue_prompt_record(*, scope: Mapping[str, Any], queue_id: str) -> dict[str, Any]: +def requeue_prompt_record( + *, + scope: Mapping[str, Any], + queue_id: str, + current_server_instance_id: str | None = None, +) -> dict[str, Any]: + """Return an interrupted record to FIFO without stealing a live lease. + + Legacy claimed rows have no durable conversation fence and remain directly + restartable. A server-bound row may be released only after the one-server + runtime identity has changed, which is the bounded restart signal for this + architecture. The captured attempt and server identity are included in the + update as a compare-and-set guard against a concurrent rebind. + """ + queue_id_clean = _coerce_scope_value(queue_id, field="queue_id") now = _now() canonical_scope = _scope_query(scope) - doc = _collection().find_one_and_update( + coll = _collection() + existing = coll.find_one( { **_compatible_scope_query(scope), "queue_id": queue_id_clean, "status": STATUS_IN_PROGRESS, + } + ) + if existing is None: + raise _transition_not_found_error( + scope=scope, + queue_id=queue_id_clean, + expected_statuses=[STATUS_IN_PROGRESS], + fallback_message="in-progress prompt was not found", + ) + + transition_guard: dict[str, Any] = {} + if existing.get("active_conversation_key"): + current_instance = _coerce_scope_value( + current_server_instance_id, + field="current_server_instance_id", + required=False, + ) + bound_instance = _coerce_scope_value( + existing.get("server_instance_id"), + field="server_instance_id", + required=False, + ) + if ( + not current_instance + or not bound_instance + or current_instance == bound_instance + ): + raise ConversationTurnAlreadyActive( + "the server-bound turn is still owned by its executing server", + queue_id=queue_id_clean, + ) + transition_guard = { + "active_conversation_key": existing.get("active_conversation_key"), + "attempt_id": existing.get("attempt_id"), + "server_instance_id": bound_instance, + } + else: + transition_guard = {"active_conversation_key": {"$exists": False}} + + requeue_query = { + **_compatible_scope_query(scope), + "queue_id": queue_id_clean, + "status": STATUS_IN_PROGRESS, + **transition_guard, + } + requeue_update = { + "$set": { + **canonical_scope, + "status": STATUS_QUEUED, + "queued_at": now, + "updated_at": now, + "claimed_at": None, + "completed_at": None, + "last_error": None, + "source": "restart", }, - { - "$set": { - **canonical_scope, - "status": STATUS_QUEUED, - "queued_at": now, - "updated_at": now, - "claimed_at": None, - "completed_at": None, - "last_error": None, - "source": "restart", - }, - "$unset": { - "stale_advisory": "", - "stale_advisory_at": "", - "stale_advisory_reason": "", - "reconciliation_required": "", - }, + "$unset": { + "active_conversation_key": "", + "client_request_id": "", + "attempt_id": "", + "server_instance_id": "", + "lease_acquired_at": "", + "lease_heartbeat_at": "", + "stale_advisory": "", + "stale_advisory_at": "", + "stale_advisory_reason": "", + "reconciliation_required": "", }, - return_document=ReturnDocument.AFTER, + } + doc = _requeue_with_bounded_slots( + coll=coll, + query=requeue_query, + update=requeue_update, + user_concept_id=str(canonical_scope["user_concept_id"]), ) record = serialise_queue_record(doc) if record is None: @@ -587,17 +1095,30 @@ def finish_prompt_record( queue_id: str, status: str = STATUS_COMPLETED, error: str | None = None, + attempt_id: str | None = None, ) -> dict[str, Any]: if status not in TERMINAL_STATUSES: raise InvalidChatPromptQueueInput("status must be a terminal queue status") queue_id_clean = _coerce_scope_value(queue_id, field="queue_id") now = _now() canonical_scope = _scope_query(scope) - doc = _collection().find_one_and_update( + attempt_id_clean = _coerce_scope_value( + attempt_id, + field="attempt_id", + required=False, + ) + ownership_guard: dict[str, Any] = ( + {"attempt_id": attempt_id_clean} + if attempt_id_clean + else {"active_conversation_key": {"$exists": False}} + ) + coll = _collection() + doc = coll.find_one_and_update( { **_compatible_scope_query(scope), "queue_id": queue_id_clean, "status": {"$in": ACTIVE_STATUSES}, + **ownership_guard, }, { "$set": { @@ -613,6 +1134,12 @@ def finish_prompt_record( ), }, "$unset": { + "active_conversation_key": "", + "queued_global_slot": "", + "queued_user_slot": "", + "server_instance_id": "", + "lease_acquired_at": "", + "lease_heartbeat_at": "", "stale_advisory": "", "stale_advisory_at": "", "stale_advisory_reason": "", @@ -622,6 +1149,20 @@ def finish_prompt_record( return_document=ReturnDocument.AFTER, ) record = serialise_queue_record(doc) + if record is None and attempt_id_clean: + # A retry after an unknown Mongo acknowledgement must recognise the + # exact already-committed attempt instead of stranding the in-process + # token and its capacity reservation. + record = serialise_queue_record( + coll.find_one( + { + **_compatible_scope_query(scope), + "queue_id": queue_id_clean, + "attempt_id": attempt_id_clean, + "status": status, + } + ) + ) if record is None: raise _transition_not_found_error( scope=scope, @@ -637,11 +1178,31 @@ def cancel_prompt_record(*, scope: Mapping[str, Any], queue_id: str) -> dict[str now = _now() canonical_scope = _scope_query(scope) - doc = _collection().find_one_and_update( + coll = _collection() + existing = coll.find_one( + {**_compatible_scope_query(scope), "queue_id": queue_id_clean} + ) + if ( + existing is not None + and existing.get("status") == STATUS_IN_PROGRESS + and existing.get("active_conversation_key") + ): + raise ConversationTurnAlreadyActive( + "an executing turn must be cancelled through its bound task", + queue_id=queue_id_clean, + ) + + doc = coll.find_one_and_update( { **_compatible_scope_query(scope), "queue_id": queue_id_clean, - "status": {"$in": ACTIVE_STATUSES}, + "$or": [ + {"status": STATUS_QUEUED}, + { + "status": STATUS_IN_PROGRESS, + "active_conversation_key": {"$exists": False}, + }, + ], }, { "$set": { @@ -652,6 +1213,12 @@ def cancel_prompt_record(*, scope: Mapping[str, Any], queue_id: str) -> dict[str "last_error": None, }, "$unset": { + "active_conversation_key": "", + "queued_global_slot": "", + "queued_user_slot": "", + "server_instance_id": "", + "lease_acquired_at": "", + "lease_heartbeat_at": "", "stale_advisory": "", "stale_advisory_at": "", "stale_advisory_reason": "", @@ -663,7 +1230,7 @@ def cancel_prompt_record(*, scope: Mapping[str, Any], queue_id: str) -> dict[str if doc is None: # Failed records keep their original completion time and diagnostic # evidence when explicitly dismissed. - doc = _collection().find_one_and_update( + doc = coll.find_one_and_update( { **_compatible_scope_query(scope), "queue_id": queue_id_clean, diff --git a/src/backend/services/conversation_turn_admission_service.py b/src/backend/services/conversation_turn_admission_service.py new file mode 100644 index 000000000..aa51bc58b --- /dev/null +++ b/src/backend/services/conversation_turn_admission_service.py @@ -0,0 +1,489 @@ +"""Bounded admission and durable single-flight for conversation turns. + +The persisted chat prompt queue owns correlation and the cross-process +conversation fence. Small in-process counters bound the current server's +global and per-actor live work without turning unrelated conversations into a +single FIFO. They are deliberately advisory to a future multi-process +dispatcher; the durable unique conversation key remains the correctness fence. +""" + +from __future__ import annotations + +import hashlib +import logging +import os +import socket +import threading +import time +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from . import chat_prompt_queue_service + +_logger = logging.getLogger(__name__) + +_TERMINALISATION_RETRY_INITIAL_DELAY_SEC = 0.5 +_TERMINALISATION_RETRY_MAX_DELAY_SEC = 30.0 +_DEFAULT_WAITRESS_THREADS = 32 +_DEFAULT_HTTP_HEADROOM_THREADS = 8 + + +def _positive_int_env(name: str, default: int) -> int: + try: + return max(1, int(os.getenv(name, str(default)))) + except (TypeError, ValueError): + return default + + +def _configured_waitress_threads() -> int: + return _positive_int_env("VON_WAITRESS_THREADS", _DEFAULT_WAITRESS_THREADS) + + +def _default_global_turn_limit() -> int: + """Leave request threads available for health, progress, and cancellation.""" + + waitress_threads = _configured_waitress_threads() + requested_headroom = _positive_int_env( + "VON_TURN_HTTP_HEADROOM_THREADS", + _DEFAULT_HTTP_HEADROOM_THREADS, + ) + effective_headroom = min(requested_headroom, max(0, waitress_threads - 1)) + return max(1, waitress_threads - effective_headroom) + + +SERVER_INSTANCE_ID = f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:12]}" + + +class ConversationTurnAdmissionError(RuntimeError): + """Base class for typed, retryable turn-admission failures.""" + + error_code = "conversation_turn_admission_failed" + status_code = 503 + + def __init__(self, message: str, *, queue_id: str | None = None) -> None: + super().__init__(message) + self.queue_id = queue_id + + +class ConversationTurnActive(ConversationTurnAdmissionError): + """Another request is already executing against this conversation.""" + + error_code = "conversation_turn_active" + status_code = 409 + + +class ConversationTurnCapacityReached(ConversationTurnAdmissionError): + """The bounded server or actor capacity is currently exhausted.""" + + error_code = "turn_capacity_reached" + status_code = 429 + + +@dataclass +class ConversationTurnAdmissionToken: + """Exact admission handle; release is idempotent through the service.""" + + token_id: str + queue_id: str | None + attempt_id: str | None + user_concept_id: str + organisation_concept_id: str | None + namespace: str | None + conversation_key: str + client_request_id: str + released: bool = False + pending_terminal_status: str | None = None + pending_terminal_error: str | None = None + + @property + def scope(self) -> dict[str, str | None]: + return { + "user_concept_id": self.user_concept_id, + "organisation_concept_id": self.organisation_concept_id, + "namespace": self.namespace, + } + + +@dataclass +class _TerminalisationRetry: + """One exact terminal disposition awaiting durable reconciliation.""" + + token: ConversationTurnAdmissionToken + status: str + error: str | None + failure_count: int + retry_at_monotonic: float + + +def build_conversation_key( + *, + owner_user_id: str, + history_namespace: str, + conversation_session_id: str, +) -> str: + """Return a content-free stable key for one canonical conversation carrier.""" + + material = "\x1f".join( + ( + str(owner_user_id or "").strip(), + str(history_namespace or "").strip(), + str(conversation_session_id or "").strip(), + ) + ) + if not all(material_part for material_part in material.split("\x1f")): + raise ValueError("complete canonical conversation identity is required") + return hashlib.sha256(material.encode("utf-8")).hexdigest() + + +class ConversationTurnAdmissionService: + """Coordinate bounded live turns while preserving cross-conversation parallelism.""" + + def __init__( + self, + *, + per_user_limit: int | None = None, + global_limit: int | None = None, + terminalisation_retry_initial_delay_sec: float = ( + _TERMINALISATION_RETRY_INITIAL_DELAY_SEC + ), + terminalisation_retry_max_delay_sec: float = ( + _TERMINALISATION_RETRY_MAX_DELAY_SEC + ), + ) -> None: + self._per_user_limit = per_user_limit or _positive_int_env( + "VON_MAX_ACTIVE_TURNS_PER_USER", 4 + ) + self._configured_http_threads = _configured_waitress_threads() + self._global_limit = global_limit or _positive_int_env( + "VON_MAX_ACTIVE_TURNS_GLOBAL", _default_global_turn_limit() + ) + self._lock = threading.RLock() + self._active_tokens: dict[str, ConversationTurnAdmissionToken] = {} + self._active_by_user: dict[str, int] = {} + self._ephemeral_conversation_tokens: dict[str, str] = {} + self._reserved_global = 0 + self._terminalisation_retry_initial_delay_sec = max( + 0.01, + float(terminalisation_retry_initial_delay_sec), + ) + self._terminalisation_retry_max_delay_sec = max( + self._terminalisation_retry_initial_delay_sec, + float(terminalisation_retry_max_delay_sec), + ) + self._terminalisation_retries: dict[str, _TerminalisationRetry] = {} + self._terminalisation_retry_failure_counts: dict[str, int] = {} + self._terminalisation_retry_condition = threading.Condition(self._lock) + self._terminalisation_retry_thread: threading.Thread | None = None + self._terminalisation_retry_shutdown = False + + def _ensure_terminalisation_retry_thread_locked(self) -> None: + thread = self._terminalisation_retry_thread + if thread is not None and thread.is_alive(): + return + self._terminalisation_retry_shutdown = False + thread = threading.Thread( + target=self._terminalisation_retry_loop, + name="conversation_turn_terminalisation_retry", + daemon=True, + ) + self._terminalisation_retry_thread = thread + thread.start() + + def _schedule_terminalisation_retry_locked( + self, + token: ConversationTurnAdmissionToken, + *, + status: str, + error: str | None, + ) -> None: + failure_count = ( + self._terminalisation_retry_failure_counts.get(token.token_id, 0) + 1 + ) + self._terminalisation_retry_failure_counts[token.token_id] = failure_count + delay = min( + self._terminalisation_retry_max_delay_sec, + self._terminalisation_retry_initial_delay_sec + * (2 ** min(failure_count - 1, 10)), + ) + self._terminalisation_retries[token.token_id] = _TerminalisationRetry( + token=token, + status=status, + error=error, + failure_count=failure_count, + retry_at_monotonic=time.monotonic() + delay, + ) + self._ensure_terminalisation_retry_thread_locked() + self._terminalisation_retry_condition.notify_all() + + def _terminalisation_retry_loop(self) -> None: + while True: + with self._terminalisation_retry_condition: + while not self._terminalisation_retry_shutdown: + if not self._terminalisation_retries: + self._terminalisation_retry_condition.wait() + continue + retry = min( + self._terminalisation_retries.values(), + key=lambda item: item.retry_at_monotonic, + ) + wait_seconds = retry.retry_at_monotonic - time.monotonic() + if wait_seconds > 0: + self._terminalisation_retry_condition.wait(timeout=wait_seconds) + continue + # Remove the due item before retrying. A failed release puts + # it back with backoff; a successful or concurrent release + # leaves no stale item that could spin at a past deadline. + self._terminalisation_retries.pop(retry.token.token_id, None) + break + else: + return + + try: + self.release( + retry.token, + status=retry.status, + error=retry.error, + ) + except Exception as exc: # noqa: BLE001 - retry loop must survive store errors + _logger.warning( + "Turn terminalisation retry %s failed for queue_id=%s: %s", + retry.failure_count, + retry.token.queue_id, + exc, + ) + + def shutdown(self, *, wait: bool = True, timeout: float = 2.0) -> None: + """Stop this instance's lazy retry worker (primarily for tests).""" + + with self._terminalisation_retry_condition: + self._terminalisation_retry_shutdown = True + self._terminalisation_retry_condition.notify_all() + thread = self._terminalisation_retry_thread + if wait and thread is not None and thread is not threading.current_thread(): + thread.join(timeout=max(0.0, float(timeout))) + + def _reserve_capacity(self, user_concept_id: str) -> None: + with self._lock: + global_active = len(self._active_tokens) + self._reserved_global + user_active = self._active_by_user.get(user_concept_id, 0) + if global_active >= self._global_limit: + raise ConversationTurnCapacityReached( + "Von is at its current active-turn capacity" + ) + if user_active >= self._per_user_limit: + raise ConversationTurnCapacityReached( + "This user is at the current active-turn capacity" + ) + # Reserve before the durable bind so racing requests in this + # process cannot all pass a count-then-act check. + self._active_by_user[user_concept_id] = user_active + 1 + self._reserved_global += 1 + + def _release_user_capacity( + self, + user_concept_id: str, + *, + reserved: bool, + ) -> None: + with self._lock: + if reserved: + self._reserved_global = max(0, self._reserved_global - 1) + remaining = self._active_by_user.get(user_concept_id, 0) - 1 + if remaining > 0: + self._active_by_user[user_concept_id] = remaining + else: + self._active_by_user.pop(user_concept_id, None) + + def acquire( + self, + *, + scope: Mapping[str, Any], + prompt_raw: str, + session_id: str, + session_name: str | None, + client_request_id: str, + conversation_key: str, + queue_id: str | None = None, + attempt_id: str | None = None, + ) -> ConversationTurnAdmissionToken: + """Acquire capacity and the durable conversation fence without waiting.""" + + canonical_scope = chat_prompt_queue_service.build_queue_scope( + user_concept_id=str(scope.get("user_concept_id") or ""), + organisation_concept_id=scope.get("organisation_concept_id"), + namespace=scope.get("namespace"), + ) + user_id = str(canonical_scope["user_concept_id"] or "") + self._reserve_capacity(user_id) + try: + bound_queue_id = str(queue_id or "").strip() + if not bound_queue_id: + try: + record = chat_prompt_queue_service.create_queue_record( + scope=canonical_scope, + prompt_raw=prompt_raw, + session_id=session_id, + session_name=session_name, + status=chat_prompt_queue_service.STATUS_QUEUED, + source="server_direct", + client_request_id=client_request_id, + attempt_id=attempt_id, + conversation_key=conversation_key, + ) + except chat_prompt_queue_service.ChatPromptQueueCapacityReached as exc: + raise ConversationTurnCapacityReached(str(exc)) from exc + bound_queue_id = str(record["queue_id"]) + try: + bound = chat_prompt_queue_service.bind_queue_record_to_turn( + scope=canonical_scope, + queue_id=bound_queue_id, + client_request_id=client_request_id, + conversation_key=conversation_key, + server_instance_id=SERVER_INSTANCE_ID, + attempt_id=attempt_id, + ) + except chat_prompt_queue_service.ConversationTurnAlreadyActive as exc: + raise ConversationTurnActive( + str(exc), + queue_id=bound_queue_id, + ) from exc + + token = ConversationTurnAdmissionToken( + token_id=str(uuid.uuid4()), + queue_id=bound_queue_id, + attempt_id=str(bound.get("attempt_id") or attempt_id or ""), + user_concept_id=user_id, + organisation_concept_id=canonical_scope.get("organisation_concept_id"), + namespace=canonical_scope.get("namespace"), + conversation_key=conversation_key, + client_request_id=client_request_id, + ) + with self._lock: + self._active_tokens[token.token_id] = token + self._reserved_global = max(0, self._reserved_global - 1) + return token + except Exception: + self._release_user_capacity(user_id, reserved=True) + raise + + def acquire_ephemeral( + self, + *, + actor_capacity_key: str, + conversation_key: str, + client_request_id: str, + ) -> ConversationTurnAdmissionToken: + """Bound an unpersisted compatibility turn without bypassing capacity. + + Anonymous or otherwise non-durable turns do not gain a prompt queue + authority record, but they still count towards global/per-browser + capacity and retain in-process single-flight for their conversation. + """ + + actor_key = str(actor_capacity_key or "").strip() + if not actor_key: + raise ValueError("actor capacity key is required") + self._reserve_capacity(actor_key) + try: + with self._lock: + if conversation_key in self._ephemeral_conversation_tokens: + raise ConversationTurnActive( + "another turn is active for this conversation" + ) + token = ConversationTurnAdmissionToken( + token_id=str(uuid.uuid4()), + queue_id=None, + attempt_id=None, + user_concept_id=actor_key, + organisation_concept_id=None, + namespace=None, + conversation_key=conversation_key, + client_request_id=client_request_id, + ) + self._active_tokens[token.token_id] = token + self._ephemeral_conversation_tokens[conversation_key] = token.token_id + self._reserved_global = max(0, self._reserved_global - 1) + return token + except Exception: + self._release_user_capacity(actor_key, reserved=True) + raise + + def release( + self, + token: ConversationTurnAdmissionToken, + *, + status: str, + error: str | None = None, + ) -> None: + """Terminalise the durable record and release capacity exactly once.""" + + with self._lock: + live_token = self._active_tokens.get(token.token_id) + if live_token is None or token.released: + return + # The first terminal observation is the exact disposition retried + # after an unknown or unavailable persistence acknowledgement. + if token.pending_terminal_status is None: + token.pending_terminal_status = status + token.pending_terminal_error = error + status = token.pending_terminal_status + error = token.pending_terminal_error + token.released = True + try: + if token.queue_id: + chat_prompt_queue_service.finish_prompt_record( + scope=token.scope, + queue_id=token.queue_id, + status=status, + error=error, + attempt_id=token.attempt_id, + ) + except Exception: + # Preserve the live token so teardown or an explicit recovery path + # can retry durable terminalisation. Releasing process capacity + # before the persisted conversation fence clears would make the + # server appear healthier than its actual admission state. + with self._lock: + token.released = False + self._schedule_terminalisation_retry_locked( + token, + status=status, + error=error, + ) + raise + + with self._lock: + self._active_tokens.pop(token.token_id, None) + self._terminalisation_retries.pop(token.token_id, None) + self._terminalisation_retry_failure_counts.pop(token.token_id, None) + self._terminalisation_retry_condition.notify_all() + if ( + self._ephemeral_conversation_tokens.get(token.conversation_key) + == token.token_id + ): + self._ephemeral_conversation_tokens.pop(token.conversation_key, None) + self._release_user_capacity(token.user_concept_id, reserved=False) + + def snapshot(self) -> dict[str, Any]: + """Return content-free live capacity telemetry.""" + + with self._lock: + return { + "schema_version": "conversation_turn_admission.v1", + "server_instance_id": SERVER_INSTANCE_ID, + "active_global": len(self._active_tokens), + "pending_admission": self._reserved_global, + "global_limit": self._global_limit, + "per_user_limit": self._per_user_limit, + "configured_http_threads": self._configured_http_threads, + "available_http_thread_headroom": max( + 0, + self._configured_http_threads - self._global_limit, + ), + "active_user_count": len(self._active_by_user), + } + + +conversation_turn_admission_service = ConversationTurnAdmissionService() diff --git a/src/backend/services/tool_progress_store_service.py b/src/backend/services/tool_progress_store_service.py index 9e75be939..459631c53 100644 --- a/src/backend/services/tool_progress_store_service.py +++ b/src/backend/services/tool_progress_store_service.py @@ -462,6 +462,36 @@ def fetch_tool_progress_state( return copy.deepcopy(dict(payload)) +def find_tool_progress_scope_keys(*, request_id: str, limit: int = 4) -> list[str]: + """Return bounded scope identities for mismatch rejection, never payload data.""" + + if not isinstance(request_id, str) or not request_id.strip(): + return [] + coll = _get_collection() + if coll is None: + return [] + _ensure_indexes() + try: + docs = coll.find( + {"request_id": request_id.strip()}, + {"_id": 0, "scope_key": 1}, + ).limit(max(1, min(int(limit or 4), 16))) + return [ + str(doc.get("scope_key")) + for doc in docs + if isinstance(doc, Mapping) + and isinstance(doc.get("scope_key"), str) + and doc.get("scope_key") + ] + except Exception as exc: # pragma: no cover - DB backend dependent + logger.debug( + "[tool_progress_store] Failed to resolve request scope request=%s: %s", + request_id, + exc, + ) + return [] + + def delete_tool_progress_state(*, scope_key: str, request_id: str) -> bool: coll = _get_collection() if coll is None: diff --git a/src/backend/services/window_session_context_service.py b/src/backend/services/window_session_context_service.py index 45dae5fde..596f324bf 100644 --- a/src/backend/services/window_session_context_service.py +++ b/src/backend/services/window_session_context_service.py @@ -36,6 +36,10 @@ class WindowSessionOwnershipError(PermissionError): """Raised when a live window-session ID is used by a different actor.""" +class WindowSessionContextUnavailable(PermissionError): + """Raised when an actor-bound tab selector has no usable server binding.""" + + def _normalise_owner_id(value: Optional[str]) -> Optional[str]: if not isinstance(value, str): return None @@ -338,6 +342,8 @@ def get_effective_context( window_session_id: Optional[str], flask_session: dict, user_id: Optional[str], + *, + require_known_window: bool = False, ) -> Dict[str, Any]: """ Get the effective context combining window session (if present) and Flask session. @@ -351,6 +357,7 @@ def get_effective_context( Returns dict with: user_id, organisation_id, role, namespace, chat_session_id """ + def _window_context_has_authoritative_scope(ctx: WindowSessionContext) -> bool: return bool( ( @@ -384,6 +391,11 @@ def _window_context_has_authoritative_scope(ctx: WindowSessionContext) -> bool: "source": "window_session", } + if require_known_window: + raise WindowSessionContextUnavailable( + "window_session_context_unavailable" + ) + flask_org = flask_session.get("organisation_concept_id") flask_role = flask_session.get("role_in_org") flask_namespace = flask_session.get("namespace") @@ -407,7 +419,14 @@ def _window_context_has_authoritative_scope(ctx: WindowSessionContext) -> bool: "source": "window_session", } - # Fall back to Flask session + if window_session_id and require_known_window: + # Missing, expired and other-actor window selectors are intentionally + # indistinguishable. Falling through to the browser-wide Flask org + # could execute an authorised action in the wrong tab's organisation. + raise WindowSessionContextUnavailable("window_session_context_unavailable") + + # Fall back to Flask session only for compatibility callers that did not + # present a window selector (or explicitly opted into non-strict lookup). return { "user_id": user_id, "organisation_id": flask_session.get("organisation_concept_id"), @@ -425,7 +444,9 @@ def delete_window_context_if_owned( """Invalidate the current actor's per-window context, if one exists.""" if not isinstance(window_session_id, str) or not window_session_id.strip(): return False - return get_window_session_store().delete_if_owned(window_session_id.strip(), user_id) + return get_window_session_store().delete_if_owned( + window_session_id.strip(), user_id + ) def delete_all_window_contexts_owned_by(user_id: Optional[str]) -> int: diff --git a/src/frontend/web/von_interface/static/js/apiService.js b/src/frontend/web/von_interface/static/js/apiService.js index 5e0c94628..4bfe0cf96 100644 --- a/src/frontend/web/von_interface/static/js/apiService.js +++ b/src/frontend/web/von_interface/static/js/apiService.js @@ -5,45 +5,54 @@ // This allows different windows to have different organisation contexts without // interfering with each other. -const WINDOW_SESSION_KEY = 'von_window_session_id'; const WINDOW_SESSION_HEADER = 'X-Von-Window-Session'; import { getSessionScopedOrgId } from './utils/sessionScopedStorage.js'; import { parseStoredContextValue } from './utils/runtimeIdentityBootstrap.js'; +import { + createWindowSessionIdentityCoordinator, + WINDOW_SESSION_KEY +} from './utils/windowSessionIdentity.js'; + +const windowSessionIdentityCoordinator = createWindowSessionIdentityCoordinator(); /** * Get or generate a unique window session ID. * This ID is stored in sessionStorage (window-scoped, not shared across tabs). */ function getWindowSessionId() { - let sessionId = sessionStorage.getItem(WINDOW_SESSION_KEY); - if (!sessionId) { - // Generate a new UUID-like session ID - const randomId = - (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') - ? crypto.randomUUID().replace(/-/g, '') - : `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 10)}`; - sessionId = 'ws_' + randomId; - sessionStorage.setItem(WINDOW_SESSION_KEY, sessionId); - console.debug('[windowSession] Generated new window session:', sessionId); - } - return sessionId; + return windowSessionIdentityCoordinator.getWindowSessionId(); +} + +/** + * Resolve copied sessionStorage IDs before the tab's first actor-scoped call. + * Existing tabs retain their ID; only a newly probing colliding document + * rotates to a fresh server window context. + */ +async function ensureUniqueWindowSessionId() { + return windowSessionIdentityCoordinator.ensureUniqueWindowSessionId(); } /** * Builds the standard headers object for fetch requests. * Includes Content-Type and the window session header. */ -function buildHeaders(extraHeaders = {}) { +async function buildHeaders(extraHeaders = {}) { + const windowSessionId = await ensureUniqueWindowSessionId(); return { 'Content-Type': 'application/json', - [WINDOW_SESSION_HEADER]: getWindowSessionId(), + [WINDOW_SESSION_HEADER]: windowSessionId, ...extraHeaders }; } // Expose for debugging/testing -export { getWindowSessionId, WINDOW_SESSION_HEADER, WINDOW_SESSION_KEY }; +export { + ensureUniqueWindowSessionId, + getWindowSessionId, + WINDOW_SESSION_HEADER, + WINDOW_SESSION_KEY +}; // ============================================================================ // Standard HTTP Helpers with Window Session Support @@ -52,7 +61,7 @@ export { getWindowSessionId, WINDOW_SESSION_HEADER, WINDOW_SESSION_KEY }; export async function getJson(url) { const res = await fetch(url, { method: 'GET', - headers: buildHeaders() + headers: await buildHeaders() }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return res.json(); @@ -67,7 +76,7 @@ export async function getJsonDetailed(url, options = {}) { const res = await fetch(url, { method, - headers: buildHeaders(extraHeaders), + headers: await buildHeaders(extraHeaders), ...fetchOptions }); @@ -104,7 +113,7 @@ export async function getJsonDetailed(url, options = {}) { export async function postJson(url, data) { const res = await fetch(url, { method: 'POST', - headers: buildHeaders(), + headers: await buildHeaders(), body: data ? JSON.stringify(data) : '{}' }); if (!res.ok) throw new Error(`HTTP ${res.status}`); @@ -120,7 +129,7 @@ export async function postJsonDetailed(url, data, options = {}) { const res = await fetch(url, { method, - headers: buildHeaders(extraHeaders), + headers: await buildHeaders(extraHeaders), body: JSON.stringify(data || {}), ...fetchOptions }); @@ -219,7 +228,7 @@ export async function putJson(url, data, opts = {}) { const extraHeaders = opts.headers || {}; const res = await fetch(url, { method: 'PUT', - headers: buildHeaders(extraHeaders), + headers: await buildHeaders(extraHeaders), body: JSON.stringify(data) }); if (!res.ok) throw new Error(`HTTP ${res.status}`); @@ -231,7 +240,7 @@ export async function patchJson(url, data, opts = {}) { const extraHeaders = opts.headers || {}; const res = await fetch(url, { method: 'PATCH', - headers: buildHeaders(extraHeaders), + headers: await buildHeaders(extraHeaders), body: JSON.stringify(data || {}) }); if (!res.ok) throw new Error(`HTTP ${res.status}`); @@ -241,7 +250,7 @@ export async function patchJson(url, data, opts = {}) { export async function deleteJson(url, data) { const options = { method: 'DELETE', - headers: buildHeaders() + headers: await buildHeaders() }; if (data !== undefined) { options.body = JSON.stringify(data); @@ -309,7 +318,7 @@ export async function annotateTurn(payload) { console.info('[annotations] annotateTurn request', payload); const res = await fetch('/api/annotations/turn', { method: 'POST', - headers: buildHeaders(), + headers: await buildHeaders(), body: JSON.stringify(payload) }); const text = await res.text(); @@ -340,7 +349,7 @@ export async function acceptAnnotation(payload) { const res = await fetch('/api/annotations/accept', { method: 'POST', - headers: buildHeaders(), + headers: await buildHeaders(), body: JSON.stringify(payload) }); if (!res.ok) throw new Error(`HTTP ${res.status}`); @@ -356,7 +365,7 @@ export async function revokeAnnotation(payload) { try { const res = await fetch('/api/annotations/revoke', { method: 'POST', - headers: buildHeaders(), + headers: await buildHeaders(), body: JSON.stringify(payload) }); if (!res.ok) throw new Error(`HTTP ${res.status}`); diff --git a/src/frontend/web/von_interface/static/js/chatTab.js b/src/frontend/web/von_interface/static/js/chatTab.js index f767562e1..6cbd3f8a3 100644 --- a/src/frontend/web/von_interface/static/js/chatTab.js +++ b/src/frontend/web/von_interface/static/js/chatTab.js @@ -73,18 +73,29 @@ import { // Helper to build fetch headers with window session context (JVNAUTOSCI-1011) // JVNAUTOSCI-1651: Include X-User-Concept-ID so backend can resolve user // identity even when Flask session state is absent (e.g. after server restart). -function buildChatFetchHeaders(extraHeaders = {}) { +function buildChatFetchHeaders(extraHeaders = {}, { includeLegacyUserHeader = true } = {}) { const headers = { [WINDOW_SESSION_HEADER]: getWindowSessionId(), ...extraHeaders }; const userConceptId = getCurrentUserConceptId(); - if (userConceptId) { + if (includeLegacyUserHeader && userConceptId) { headers['X-User-Concept-ID'] = userConceptId; } return headers; } +async function repairCurrentWindowOrganisationBinding() { + const context = getUserContext(); + if (!context?.user_id) { + return false; + } + await postJson('/von/api/session/set_organisation', { + organisation_concept_id: context.org_id || null + }); + return true; +} + const LOCAL_MODEL_UNAVAILABLE_CHAT_MESSAGE = 'No usable model configured. Choose an Ollama model in Settings or enable premium model use.'; function getUnavailableLocalModelPreferenceForChat() { @@ -1921,8 +1932,10 @@ const THINKING_PROGRESS_POLL_FETCH_TIMEOUT_MS = 30_000; const THINKING_PROGRESS_TIMEOUT_VISIBLE_AFTER_MS = 8_000; const THINKING_PROGRESS_TIMEOUTS_BEFORE_VISIBLE = 2; const FOREGROUND_TASK_RESULT_POLL_FETCH_TIMEOUT_MS = 8_000; -const FOREGROUND_TASK_RESULT_POLL_INITIAL_DELAY_MS = 750; -const FOREGROUND_TASK_RESULT_POLL_MAX_DELAY_MS = 5_000; +// The synchronous response is the primary foreground delivery path. Task +// polling is delayed recovery, not a second eager progress channel per turn. +const FOREGROUND_TASK_RESULT_POLL_INITIAL_DELAY_MS = 10_000; +const FOREGROUND_TASK_RESULT_POLL_MAX_DELAY_MS = 60_000; const THINKING_DIAGNOSTICS_EXPORT_EVENT_LIMIT = 40; const THINKING_DIAGNOSTICS_EXPORT_PHASE_HISTORY_LIMIT = 80; const THINKING_TERMINAL_PROGRESS_STATUSES = new Set([ @@ -5684,6 +5697,10 @@ function getLiveChatRequestForSession(sessionId = activeChatSessionId) { return liveChatRequestsBySession.get(getChatRequestSessionKey(sessionId)) || null; } +function hasLiveChatRequestForSession(sessionId = activeChatSessionId) { + return Boolean(getLiveChatRequestForSession(sessionId)); +} + function getFinishedThinkingCardForSession(sessionId = activeChatSessionId) { return finishedThinkingCardsBySession.get(getChatRequestSessionKey(sessionId)) || null; } @@ -5720,10 +5737,38 @@ function getQueuedChatPromptCountForSession(sessionId = activeChatSessionId) { return queuedChatPrompts.filter((entry) => entry?.sessionKey === sessionKey).length; } +function isRestartableChatPromptQueueEntry(entry) { + if (!entry || entry.status !== CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS) { + return false; + } + if (getLiveChatRequestForPromptQueueRecord(entry.queueId)) { + return false; + } + const recordServerInstanceId = (typeof entry.serverInstanceId === 'string' && entry.serverInstanceId.trim()) + ? entry.serverInstanceId.trim() + : null; + if (!recordServerInstanceId) { + return true; + } + return Boolean( + chatPromptQueueAdmissionServerInstanceId + && recordServerInstanceId !== chatPromptQueueAdmissionServerInstanceId + ); +} + +function isRunningChatPromptQueueEntry(entry) { + return Boolean( + entry + && entry.status === CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS + && !isRestartableChatPromptQueueEntry(entry) + ); +} + function getChatPromptQueueCountsForSession(sessionId = activeChatSessionId) { const sessionKey = getChatRequestSessionKey(sessionId); const counts = { total: 0, + running: 0, queued: 0, restartable: 0, failed: 0 @@ -5735,8 +5780,10 @@ function getChatPromptQueueCountsForSession(sessionId = activeChatSessionId) { counts.total += 1; if (entry.status === CHAT_PROMPT_QUEUE_STATUS_FAILED) { counts.failed += 1; - } else if (entry.status === CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS) { + } else if (isRestartableChatPromptQueueEntry(entry)) { counts.restartable += 1; + } else if (entry.status === CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS) { + counts.running += 1; } else { counts.queued += 1; } @@ -5817,8 +5864,15 @@ function updateSendButtonForCurrentChatState() { return; } const uploadInFlight = !!getInFlightUploadStateForSession(activeChatSessionId); + const queueCounts = getChatPromptQueueCountsForSession(activeChatSessionId); + const sessionBusy = hasLiveChatRequestForSession(activeChatSessionId) + || queueCounts.running > 0 + || queueCounts.queued > 0 + || queueCounts.restartable > 0; sendButton.disabled = uploadInFlight; - sendButton.textContent = hasAnyLiveChatRequest() ? 'Queue Prompt' : 'Send Prompt'; + sendButton.textContent = sessionBusy + ? 'Queue Prompt' + : 'Send Prompt'; if (uploadInFlight) { sendButton.title = ATTACHMENT_UPLOAD_SEND_BLOCK_MESSAGE; } else { @@ -6372,6 +6426,10 @@ export function __testOnly_getThinkingProgressPollFetchTimeoutMs() { return THINKING_PROGRESS_POLL_FETCH_TIMEOUT_MS; } +export function __testOnly_getForegroundTaskResultPollInitialDelayMs() { + return FOREGROUND_TASK_RESULT_POLL_INITIAL_DELAY_MS; +} + function updateThinkingCardStatusBadge(progress, cardRoot = null) { const badgeEl = getThinkingCardStatusBadgeEl(cardRoot); if (!badgeEl) { @@ -13758,10 +13816,11 @@ function startToolUseProgressPolling(request) { { method: 'GET', signal: poll.abortController?.signal, - headers: buildChatFetchHeaders(), - // Avoid same-session request serialization against the live - // /von/generate POST; header-based scope is sufficient here. - credentials: 'omit', + // Progress is an authenticated actor/org read. The legacy + // identity header is intentionally omitted: it is neither + // authentication nor namespace authority. + headers: buildChatFetchHeaders({}, { includeLegacyUserHeader: false }), + credentials: 'same-origin', timeoutMs: THINKING_PROGRESS_POLL_FETCH_TIMEOUT_MS }); if (!resp) { @@ -14012,7 +14071,7 @@ function startForegroundTaskResultPolling(request, handlers = {}) { method: 'GET', signal: poll.abortController?.signal, headers: buildChatFetchHeaders(), - credentials: 'omit', + credentials: 'same-origin', timeoutMs: FOREGROUND_TASK_RESULT_POLL_FETCH_TIMEOUT_MS }); if (!resp || typeof resp.json !== 'function') { @@ -14027,6 +14086,17 @@ function startForegroundTaskResultPolling(request, handlers = {}) { return { resp, payload }; }; + const repairWindowContextIfNeeded = async (resp, payload) => { + const errorCode = String(payload?.error_code || payload?.error || '').trim(); + if (resp?.status !== 409 || errorCode !== 'window_context_unavailable') { + return false; + } + await repairCurrentWindowOrganisationBinding(); + poll.nextDelayMs = 250; + scheduleNextPoll(poll.nextDelayMs); + return true; + }; + const pollOnce = async () => { if ( poll.delivering @@ -14041,6 +14111,9 @@ function startForegroundTaskResultPolling(request, handlers = {}) { const { resp: statusResp, payload: statusPayload } = await fetchJson( `/von/api/task/status/${encodeURIComponent(requestId)}` ); + if (await repairWindowContextIfNeeded(statusResp, statusPayload)) { + return; + } if (!statusResp || statusResp.status === 404 || statusResp.status === 409) { poll.nextDelayMs = Math.min( FOREGROUND_TASK_RESULT_POLL_MAX_DELAY_MS, @@ -14068,6 +14141,9 @@ function startForegroundTaskResultPolling(request, handlers = {}) { const { resp: resultResp, payload: resultPayload } = await fetchJson( `/von/api/task/result/${encodeURIComponent(requestId)}` ); + if (await repairWindowContextIfNeeded(resultResp, resultPayload)) { + return; + } if (!resultResp || resultResp.status === 404 || resultResp.status === 409) { poll.nextDelayMs = Math.min( FOREGROUND_TASK_RESULT_POLL_MAX_DELAY_MS, @@ -23275,7 +23351,11 @@ const finishedThinkingCardsBySession = new Map(); let queuedChatPrompts = []; let queuedChatPromptCounter = 0; let queuedChatPromptDrainTimer = null; +let queuedChatPromptDrainPromise = null; +let queuedChatPromptDrainRequested = false; +const queuedChatPromptClaimsInFlight = new Set(); const queuedChatPromptSyncTimers = new Map(); +let chatPromptQueueAdmissionServerInstanceId = null; let selectedChatPromptQueueEntryId = null; const locallyHiddenChatPromptQueueEntryKeys = new Set(); const LEGACY_CHAT_REQUEST_SESSION_KEY = '__legacy_active_chat_session__'; @@ -23289,6 +23369,7 @@ const CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS = 'in_progress'; const CHAT_PROMPT_QUEUE_STATUS_COMPLETED = 'completed'; const CHAT_PROMPT_QUEUE_STATUS_FAILED = 'failed'; const CHAT_PROMPT_QUEUE_STATUS_CANCELLED = 'cancelled'; +const CHAT_PROMPT_QUEUE_SYNC_STORAGE_KEY = 'von:chatPromptQueueChanged:v1'; const CHAT_TTS_STORAGE_KEY = 'chatTtsEnabled'; @@ -26258,6 +26339,9 @@ function renderChatSessionTabs(sessions, activeSessionId) { if (hasLiveRequest) { activityBadge.textContent = sid === activeSessionId ? 'Thinking' : 'Background'; tab.title = `${tab.title} • ${sid === activeSessionId ? 'Thinking' : 'Background request active'}`; + } else if (queueCounts.running > 0) { + activityBadge.textContent = queueCounts.running === 1 ? 'Running' : `Running ${queueCounts.running}`; + tab.title = `${tab.title} • ${queueCounts.running} prompt${queueCounts.running === 1 ? '' : 's'} running in another tab`; } else if (queueCounts.failed > 0 && (queueCounts.queued + queueCounts.restartable) === 0) { activityBadge.textContent = queueCounts.failed === 1 ? 'Failed' : `Failed ${queueCounts.failed}`; tab.title = `${tab.title} • ${queueCounts.failed} failed queued prompt${queueCounts.failed === 1 ? '' : 's'}`; @@ -26648,7 +26732,11 @@ async function beginFocusedConversation({ focalConceptIds, mode, sessionId = nul if (!focus.length) { throw new Error('A focal concept is required to start a discussion.'); } - if (mode === 'assistant_opening' && hasAnyLiveChatRequest()) { + if ( + mode === 'assistant_opening' + && sessionId + && isChatPromptSessionBusy(sessionId, { includeQueued: true }) + ) { throw new Error('Wait for the current response before asking Von to open another discussion.'); } @@ -26670,7 +26758,7 @@ async function beginFocusedConversation({ focalConceptIds, mode, sessionId = nul } if (mode === 'assistant_opening') { - if (hasAnyLiveChatRequest()) { + if (isChatPromptSessionBusy(targetSessionId, { includeQueued: true })) { throw new Error('Wait for the current response before asking Von to open another discussion.'); } // The request owns its visible thinking/error state. Do not leave the @@ -31683,6 +31771,7 @@ function handleRealtimeConnectionsVisibilityChange() { scheduleChatSessionTabsRefresh(true); void loadIncomingInvites({ silent: true }); void refreshWorkflowStatusSnapshot({ silent: true }); + void refreshChatPromptQueueFromServer({ silent: true }); publishRealtimeConnectionTelemetry('document_visible'); } @@ -32124,6 +32213,7 @@ export function initializeChatTab() { void loadIncomingInvites({ silent: true }); startIncomingInvitePolling(); document.addEventListener('visibilitychange', handleRealtimeConnectionsVisibilityChange); + window.addEventListener('storage', handleChatPromptQueueStorageEvent); window.addEventListener('beforeunload', () => { closeSharedConversationStream(); stopWorkflowStatusStream('workflow_stream_stopped_unload'); @@ -32872,7 +32962,7 @@ function retryAssistantOpeningRequest(request) { if (request?.turnKind !== 'assistant_opening' || !sessionId || !initiationId) { return false; } - if (hasAnyLiveChatRequest()) { + if (isChatPromptSessionBusy(sessionId, { includeQueued: true })) { showToast('Wait for the current response before retrying Von’s opening.', 'info'); return false; } @@ -33142,6 +33232,14 @@ function normaliseChatPromptQueueEntry(rawEntry, fallback = {}) { localOnly: queueId ? false : rawEntry.localOnly === true, syncError: rawEntry.syncError || null, lastError: rawEntry.last_error || rawEntry.lastError || null, + clientRequestId: rawEntry.client_request_id || rawEntry.clientRequestId || fallback.clientRequestId || null, + attemptId: rawEntry.attempt_id || rawEntry.attemptId || fallback.attemptId || null, + serverInstanceId: rawEntry.server_instance_id || rawEntry.serverInstanceId || fallback.serverInstanceId || null, + leaseAcquiredAt: rawEntry.lease_acquired_at || rawEntry.leaseAcquiredAt || fallback.leaseAcquiredAt || null, + leaseHeartbeatAt: rawEntry.lease_heartbeat_at || rawEntry.leaseHeartbeatAt || fallback.leaseHeartbeatAt || null, + staleAdvisory: rawEntry.stale_advisory === true || rawEntry.staleAdvisory === true, + staleAdvisoryReason: rawEntry.stale_advisory_reason || rawEntry.staleAdvisoryReason || fallback.staleAdvisoryReason || null, + reconciliationRequired: rawEntry.reconciliation_required === true || rawEntry.reconciliationRequired === true, attemptCount: Number.isFinite(Number(rawEntry.attempt_count ?? rawEntry.attemptCount)) ? Number(rawEntry.attempt_count ?? rawEntry.attemptCount) : 0, @@ -33299,8 +33397,21 @@ async function fetchChatPromptQueueJson(path = '', options = {}) { if (!response.ok || data?.success === false) { const error = new Error(data?.error || `HTTP ${response.status}`); error.httpStatus = response.status; - error.errorCode = data?.error_code || (response.status === 401 ? 'not_authenticated' : null); + error.errorCode = data?.error_code + || (data?.error === 'window_context_unavailable' ? data.error : null) + || (response.status === 401 ? 'not_authenticated' : null); error.details = data?.details && typeof data.details === 'object' ? data.details : {}; + if ( + response.status === 409 + && error.errorCode === 'window_context_unavailable' + && options.windowContextRepairAttempted !== true + ) { + await repairCurrentWindowOrganisationBinding(); + return fetchChatPromptQueueJson(path, { + ...options, + windowContextRepairAttempted: true + }); + } throw error; } return data || {}; @@ -33343,6 +33454,28 @@ function describeChatPromptQueueApiError(error, fallbackMessage) { return fallbackMessage; } +function publishChatPromptQueueChange(reason = 'changed') { + try { + localStorage.setItem(CHAT_PROMPT_QUEUE_SYNC_STORAGE_KEY, JSON.stringify({ + changed_at_ms: Date.now(), + nonce: Math.random().toString(36).slice(2), + reason + })); + } catch (_) { + // Queue durability remains server-owned when browser storage is blocked. + } +} + +function handleChatPromptQueueStorageEvent(event) { + if (event?.key !== CHAT_PROMPT_QUEUE_SYNC_STORAGE_KEY) { + return; + } + // Storage events fire only in the other same-origin tabs. Refreshing the + // actor/window-scoped canonical list is enough to unblock that tab's FIFO; + // no prompt content or queue identifier is broadcast between documents. + void refreshChatPromptQueueFromServer({ silent: true }); +} + function mergePersistedChatPromptQueueEntry(entry) { const normalised = normaliseChatPromptQueueEntry(entry); if (!normalised) { @@ -33366,36 +33499,49 @@ function mergePersistedChatPromptQueueEntry(entry) { async function refreshChatPromptQueueFromServer(options = {}) { try { const data = await fetchChatPromptQueueJson(''); + chatPromptQueueAdmissionServerInstanceId = ( + typeof data?.turn_admission?.server_instance_id === 'string' + && data.turn_admission.server_instance_id.trim() + ) + ? data.turn_admission.server_instance_id.trim() + : null; const existingByQueueId = new Map( queuedChatPrompts .filter((entry) => entry?.queueId) .map((entry) => [entry.queueId, entry]) ); - const persistedEntries = Array.isArray(data.items) + const canonicalPersistedEntries = Array.isArray(data.items) ? data.items .map((item) => normaliseChatPromptQueueEntry( item, existingByQueueId.get(item?.queue_id) || {} )) .filter(Boolean) - .filter(isDisplayedChatPromptQueueEntry) : []; - const recentFailedEntries = Array.isArray(data.recent_failed_items) + const canonicalRecentFailedEntries = Array.isArray(data.recent_failed_items) ? data.recent_failed_items .map((item) => normaliseChatPromptQueueEntry( item, existingByQueueId.get(item?.queue_id) || {} )) .filter(Boolean) - .filter(isDisplayedChatPromptQueueEntry) : []; + const persistedEntries = canonicalPersistedEntries + .filter(isDisplayedChatPromptQueueEntry); + const recentFailedEntries = canonicalRecentFailedEntries + .filter(isDisplayedChatPromptQueueEntry); const localOnlyEntries = queuedChatPrompts.filter((entry) => entry?.localOnly === true); queuedChatPrompts = [...persistedEntries, ...localOnlyEntries, ...recentFailedEntries]; renderChatTaskQueuePanel(); refreshChatSessionTabActivityIndicators(); updateSendButtonForCurrentChatState(); - scheduleQueuedChatPromptDrain(); - return true; + if (!options.skipDrain) { + scheduleQueuedChatPromptDrain(); + } + return { + items: canonicalPersistedEntries, + recentFailedItems: canonicalRecentFailedEntries + }; } catch (error) { if (!options.silent) { console.warn('[chatTab] Unable to load persisted chat prompt queue:', error); @@ -33411,6 +33557,8 @@ async function createPersistedChatPromptQueueEntry(entry, status = CHAT_PROMPT_Q prompt_raw: entry.promptRaw, session_id: entry.sessionId || null, session_name: entry.sessionName || null, + client_request_id: entry.clientRequestId || null, + attempt_id: entry.attemptId || null, status }) }); @@ -33474,16 +33622,6 @@ async function deletePersistedChatPromptQueueEntry(entry) { return true; } -async function claimPersistedChatPromptQueueEntry(entry) { - if (!entry?.queueId) { - return entry; - } - const data = await fetchChatPromptQueueJson(`/${encodeURIComponent(entry.queueId)}/claim`, { - method: 'POST' - }); - return data.item ? normaliseChatPromptQueueEntry(data.item, entry) : entry; -} - async function requeuePersistedChatPromptQueueEntry(entry) { if (!entry?.queueId) { return entry; @@ -33494,20 +33632,6 @@ async function requeuePersistedChatPromptQueueEntry(entry) { return data.item ? normaliseChatPromptQueueEntry(data.item, entry) : entry; } -async function finishPersistedChatPromptQueueRecord(queueId, status, error = null) { - if (!(typeof queueId === 'string' && queueId.trim())) { - return false; - } - await fetchChatPromptQueueJson(`/${encodeURIComponent(queueId.trim())}/finish`, { - method: 'POST', - body: JSON.stringify({ - status, - error - }) - }); - return true; -} - function createQueuedChatPromptId() { queuedChatPromptCounter += 1; return `queued-${Date.now()}-${queuedChatPromptCounter}`; @@ -33609,6 +33733,7 @@ function ensureChatTaskQueuePanel() { renderChatTaskQueuePanel(); refreshChatSessionTabActivityIndicators(); updateSendButtonForCurrentChatState(); + publishChatPromptQueueChange('queued_prompt_restarted'); scheduleQueuedChatPromptDrain(); } catch (error) { queued.syncError = describeChatPromptQueueApiError( @@ -33642,6 +33767,7 @@ function ensureChatTaskQueuePanel() { renderChatTaskQueuePanel(); refreshChatSessionTabActivityIndicators(); updateSendButtonForCurrentChatState(); + publishChatPromptQueueChange('failed_prompt_dismissed'); } catch (error) { queued.syncError = describeChatPromptQueueApiError( error, @@ -33667,9 +33793,11 @@ function ensureChatTaskQueuePanel() { refreshChatSessionTabActivityIndicators(); updateSendButtonForCurrentChatState(); if (queued) { - void deletePersistedChatPromptQueueEntry(queued).catch((error) => { - console.warn('[chatTab] Unable to delete persisted queued prompt:', error); - }); + void deletePersistedChatPromptQueueEntry(queued) + .then(() => publishChatPromptQueueChange('queued_prompt_deleted')) + .catch((error) => { + console.warn('[chatTab] Unable to delete persisted queued prompt:', error); + }); } }); } @@ -33697,10 +33825,9 @@ function renderChatTaskQueuePanel() { let failedCount = 0; let queuedCount = 0; displayEntries.forEach((entry) => { - const liveRequest = getLiveChatRequestForPromptQueueRecord(entry.queueId); - if (liveRequest) { + if (isRunningChatPromptQueueEntry(entry)) { runningCount += 1; - } else if (entry.status === CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS) { + } else if (isRestartableChatPromptQueueEntry(entry)) { interruptedCount += 1; } else if (entry.status === CHAT_PROMPT_QUEUE_STATUS_FAILED) { failedCount += 1; @@ -33738,9 +33865,8 @@ function renderChatTaskQueuePanel() { let queuedOrdinal = 0; displayEntries.forEach((entry, index) => { - const liveRequest = getLiveChatRequestForPromptQueueRecord(entry.queueId); - const isRunning = !!liveRequest; - const isInterrupted = !isRunning && entry.status === CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS; + const isRunning = isRunningChatPromptQueueEntry(entry); + const isInterrupted = isRestartableChatPromptQueueEntry(entry); const isFailed = !isRunning && entry.status === CHAT_PROMPT_QUEUE_STATUS_FAILED; const item = document.createElement('div'); item.className = 'chat-task-queue-item'; @@ -33852,6 +33978,12 @@ async function queuePromptForLater(promptRaw, options = {}) { fileCopyConceptId: normaliseTrustedUploadedFileCopyConceptId( options.fileCopyConceptId ), + clientRequestId: (typeof options.clientRequestId === 'string' && options.clientRequestId.trim()) + ? options.clientRequestId.trim() + : null, + attemptId: (typeof options.attemptId === 'string' && options.attemptId.trim()) + ? options.attemptId.trim() + : null, status: CHAT_PROMPT_QUEUE_STATUS_QUEUED, localOnly: true }); @@ -33879,6 +34011,7 @@ async function queuePromptForLater(promptRaw, options = {}) { renderChatTaskQueuePanel(); refreshChatSessionTabActivityIndicators(); updateSendButtonForCurrentChatState(); + publishChatPromptQueueChange('queued_prompt_created'); return persisted; } } catch (error) { @@ -33892,22 +34025,42 @@ async function queuePromptForLater(promptRaw, options = {}) { return localEntry; } -async function sendQueuedChatPromptEntryAtIndex(nextIndex) { - if (hasAnyLiveChatRequest()) { - return; - } - if (!Number.isInteger(nextIndex) || nextIndex < 0 || nextIndex >= queuedChatPrompts.length) { - return; +function isChatPromptSessionBusy(sessionId, { includeQueued = false } = {}) { + const sessionKey = getChatRequestSessionKey(sessionId); + return liveChatRequestsBySession.has(sessionKey) + || queuedChatPromptClaimsInFlight.has(sessionKey) + || queuedChatPrompts.some((entry) => ( + entry?.sessionKey === sessionKey + && ( + entry.status === CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS + || (includeQueued && entry.status === CHAT_PROMPT_QUEUE_STATUS_QUEUED) + ) + )); +} + +async function sendQueuedChatPromptEntry(entryId) { + const cleanEntryId = (typeof entryId === 'string' && entryId.trim()) + ? entryId.trim() + : null; + if (!cleanEntryId) { + return false; } - let nextEntry = queuedChatPrompts[nextIndex]; + let nextEntry = getChatPromptQueueEntryById(cleanEntryId); + if (!nextEntry || isChatPromptSessionBusy(nextEntry.sessionId)) { + return false; + } + const sessionHead = getQueuedChatPromptHeadsBySession().get(nextEntry.sessionKey); + if (sessionHead?.id !== nextEntry.id) { + return false; + } if (!nextEntry || typeof nextEntry.promptRaw !== 'string' || !nextEntry.promptRaw.trim()) { - queuedChatPrompts.splice(nextIndex, 1); + queuedChatPrompts = queuedChatPrompts.filter((entry) => entry?.id !== cleanEntryId); renderChatTaskQueuePanel(); refreshChatSessionTabActivityIndicators(); updateSendButtonForCurrentChatState(); scheduleQueuedChatPromptDrain(); - return; + return false; } if (getUnavailableLocalModelPreferenceForChat()) { @@ -33916,7 +34069,7 @@ async function sendQueuedChatPromptEntryAtIndex(nextIndex) { refreshChatSessionTabActivityIndicators(); updateSendButtonForCurrentChatState(); notifyUnavailableLocalModelForChat(); - return; + return false; } // A queued turn is bound to its originating conversation, but executing it @@ -33924,71 +34077,123 @@ async function sendQueuedChatPromptEntryAtIndex(nextIndex) { // reading. `handleSendPrompt` accepts the explicit target session and keeps // all live/result rendering scoped to that session. - if (nextEntry.queueId) { - try { + const sessionKey = nextEntry.sessionKey || getChatRequestSessionKey(nextEntry.sessionId); + queuedChatPromptClaimsInFlight.add(sessionKey); + try { + if (nextEntry.queueId) { const updatedEntry = await persistQueuedPromptUpdateNow(nextEntry); nextEntry = { ...nextEntry, ...updatedEntry }; - queuedChatPrompts[nextIndex] = nextEntry; - const claimed = await claimPersistedChatPromptQueueEntry(nextEntry); - nextEntry = { - ...nextEntry, - ...claimed, - status: CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS - }; - } catch (error) { - const failedEntry = queuedChatPrompts[nextIndex] || nextEntry; - failedEntry.syncError = describeChatPromptQueueApiError( - error, - 'Could not save and claim this queued task on the server.' - ); - console.warn('[chatTab] Unable to claim queued prompt:', error); - renderChatTaskQueuePanel(); - refreshChatSessionTabActivityIndicators(); - updateSendButtonForCurrentChatState(); - return; + const updatedIndex = queuedChatPrompts.findIndex((entry) => entry?.id === cleanEntryId); + if (updatedIndex < 0) { + return false; + } + queuedChatPrompts[updatedIndex] = nextEntry; } - } - queuedChatPrompts.splice(nextIndex, 1); - renderChatTaskQueuePanel(); - refreshChatSessionTabActivityIndicators(); - updateSendButtonForCurrentChatState(); + queuedChatPrompts = queuedChatPrompts.filter((entry) => entry?.id !== cleanEntryId); + renderChatTaskQueuePanel(); + refreshChatSessionTabActivityIndicators(); + updateSendButtonForCurrentChatState(); - void handleSendPrompt({ - promptOverride: nextEntry.promptRaw, - fromQueue: true, - sessionId: nextEntry.sessionId || null, - sessionName: nextEntry.sessionName || null, - promptQueueRecordId: nextEntry.queueId || null, - fileCopyConceptId: nextEntry.fileCopyConceptId || null + // Release the short claim guard before dispatch. handleSendPrompt runs + // synchronously until it installs the live request for this session. + queuedChatPromptClaimsInFlight.delete(sessionKey); + void handleSendPrompt({ + promptOverride: nextEntry.promptRaw, + fromQueue: true, + sessionId: nextEntry.sessionId || null, + sessionName: nextEntry.sessionName || null, + promptQueueRecordId: nextEntry.queueId || null, + clientRequestId: nextEntry.clientRequestId || null, + attemptId: nextEntry.attemptId || null, + fileCopyConceptId: nextEntry.fileCopyConceptId || null + }); + return true; + } catch (error) { + const failedEntry = getChatPromptQueueEntryById(cleanEntryId) || nextEntry; + failedEntry.syncError = describeChatPromptQueueApiError( + error, + 'Could not save and claim this queued task on the server.' + ); + console.warn('[chatTab] Unable to claim queued prompt:', error); + renderChatTaskQueuePanel(); + refreshChatSessionTabActivityIndicators(); + updateSendButtonForCurrentChatState(); + return false; + } finally { + queuedChatPromptClaimsInFlight.delete(sessionKey); + } +} + +function getQueuedChatPromptHeadsBySession() { + const heads = new Map(); + queuedChatPrompts.forEach((entry) => { + if (!entry || ![ + CHAT_PROMPT_QUEUE_STATUS_QUEUED, + CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS + ].includes(entry.status)) { + return; + } + const sessionKey = entry.sessionKey || getChatRequestSessionKey(entry.sessionId); + if (!heads.has(sessionKey)) { + heads.set(sessionKey, entry); + } }); + return heads; } -async function drainQueuedChatPromptIfIdle() { - if (hasAnyLiveChatRequest() || queuedChatPrompts.length === 0) { - return; +async function drainQueuedChatPromptsForFreeSessions() { + if (queuedChatPrompts.length === 0) { + return false; } - const nextIndex = queuedChatPrompts.findIndex( - (entry) => entry?.status === CHAT_PROMPT_QUEUE_STATUS_QUEUED - ); - if (nextIndex < 0) { - return; - } + let startedAny = false; + while (true) { + const eligibleEntries = Array.from(getQueuedChatPromptHeadsBySession().values()) + .filter((entry) => ( + entry.status === CHAT_PROMPT_QUEUE_STATUS_QUEUED + && !isChatPromptSessionBusy(entry.sessionId) + )); + if (eligibleEntries.length === 0) { + return startedAny; + } - await sendQueuedChatPromptEntryAtIndex(nextIndex); + let startedThisPass = false; + for (const entry of eligibleEntries) { + const started = await sendQueuedChatPromptEntry(entry.id); + startedAny = startedAny || started; + startedThisPass = startedThisPass || started; + } + if (!startedThisPass) { + return startedAny; + } + } } function scheduleQueuedChatPromptDrain() { - if (queuedChatPromptDrainTimer !== null) { + queuedChatPromptDrainRequested = true; + if (queuedChatPromptDrainTimer !== null || queuedChatPromptDrainPromise !== null) { return; } queuedChatPromptDrainTimer = setTimeout(() => { queuedChatPromptDrainTimer = null; - void drainQueuedChatPromptIfIdle(); + queuedChatPromptDrainRequested = false; + const drainPromise = drainQueuedChatPromptsForFreeSessions() + .catch((error) => { + console.warn('[chatTab] Unable to drain queued prompts:', error); + }) + .finally(() => { + if (queuedChatPromptDrainPromise === drainPromise) { + queuedChatPromptDrainPromise = null; + } + if (queuedChatPromptDrainRequested) { + scheduleQueuedChatPromptDrain(); + } + }); + queuedChatPromptDrainPromise = drainPromise; }, 0); } @@ -34003,11 +34208,6 @@ async function handleSendPrompt(options = {}) { const selectedQueueEntry = (!fromQueue && !hasPromptOverride) ? getSelectedChatPromptQueueEntryForSend() : null; - if (selectedQueueEntry?.status === CHAT_PROMPT_QUEUE_STATUS_QUEUED && !hasAnyLiveChatRequest()) { - const selectedIndex = queuedChatPrompts.findIndex((entry) => entry?.id === selectedQueueEntry.id); - await sendQueuedChatPromptEntryAtIndex(selectedIndex); - return; - } let targetSessionId = normaliseHistorySessionId( options?.sessionId ?? selectedQueueEntry?.sessionId @@ -34020,6 +34220,12 @@ async function handleSendPrompt(options = {}) { ? selectedQueueEntry.sessionName.trim() : activeChatSessionName ); + if (selectedQueueEntry?.status === CHAT_PROMPT_QUEUE_STATUS_QUEUED) { + if (!isChatPromptSessionBusy(targetSessionId)) { + await sendQueuedChatPromptEntry(selectedQueueEntry.id); + } + return; + } const promptRaw = hasPromptOverride ? options.promptOverride : (selectedQueueEntry ? selectedQueueEntry.promptRaw : promptInput.value); @@ -34048,35 +34254,6 @@ async function handleSendPrompt(options = {}) { return; } - if (hasAnyLiveChatRequest()) { - if (fromQueue) { - scheduleQueuedChatPromptDrain(); - return; - } - if (selectedQueueEntry?.status === CHAT_PROMPT_QUEUE_STATUS_QUEUED) { - return; - } - if (!promptText) { - return; - } - const queuedFileCopyConceptId = takePendingUploadedFileCopyConceptId( - targetSessionId - ); - await queuePromptForLater(promptRaw, { - sessionId: targetSessionId, - sessionName: targetSessionName, - fileCopyConceptId: queuedFileCopyConceptId - }); - if (selectedQueueEntry) { - hideChatPromptQueueEntryLocally(selectedQueueEntry); - } - rememberLastSubmittedUserPrompt(promptRaw); - if (!selectedQueueEntry) { - setPromptComposerValue('', { promptInput }); - } - return; - } - ensureAbortButtonBound(); if (!promptText && !allowEmptyPrompt) { @@ -34118,6 +34295,27 @@ async function handleSendPrompt(options = {}) { } } + if (isChatPromptSessionBusy(targetSessionId, { includeQueued: !fromQueue })) { + if (fromQueue) { + scheduleQueuedChatPromptDrain(); + return; + } + if (!promptText) { + return; + } + const queuedFileCopyConceptId = takePendingUploadedFileCopyConceptId( + targetSessionId + ); + await queuePromptForLater(promptRaw, { + sessionId: targetSessionId, + sessionName: targetSessionName, + fileCopyConceptId: queuedFileCopyConceptId + }); + rememberLastSubmittedUserPrompt(promptRaw); + setPromptComposerValue('', { promptInput }); + return; + } + // Refresh setting in the background; default is enabled. void refreshToolUseDuringThinkingSetting(); @@ -34125,7 +34323,12 @@ async function handleSendPrompt(options = {}) { ? options.promptQueueRecordId.trim() : null; - const clientRequestId = createClientRequestId(); + const clientRequestId = (typeof options?.clientRequestId === 'string' && options.clientRequestId.trim()) + ? options.clientRequestId.trim() + : createClientRequestId(); + const attemptId = (typeof options?.attemptId === 'string' && options.attemptId.trim()) + ? options.attemptId.trim() + : createClientRequestId(); const executionContextBinding = synchroniseLlmExecutionContext(); const explicitFileCopyConceptId = normaliseTrustedUploadedFileCopyConceptId( options?.fileCopyConceptId @@ -34157,6 +34360,7 @@ async function handleSendPrompt(options = {}) { attachmentBindingReleased: false, aborted: false, clientRequestId, + attemptId, executionContextBinding, thinkingStartedAtMs: Date.now(), activityHistory: [], @@ -34177,8 +34381,10 @@ async function handleSendPrompt(options = {}) { : createPersistedChatPromptQueueEntry({ promptRaw, sessionId: targetSessionId, - sessionName: targetSessionName - }, CHAT_PROMPT_QUEUE_STATUS_IN_PROGRESS) + sessionName: targetSessionName, + clientRequestId, + attemptId + }, CHAT_PROMPT_QUEUE_STATUS_QUEUED) .then((activeRecord) => { request.promptQueueRecordId = activeRecord?.queueId || null; return activeRecord; @@ -34214,6 +34420,8 @@ async function handleSendPrompt(options = {}) { ? 'Von is opening this discussion…' : formatToolUseProgressText(buildInitialThinkingProgressPlaceholder(clientRequestId), request) ); + setLoadingIndicatorDetailHtml(renderThinkingCardBodyHTML(request), request); + updateThinkingCardMeta(request, null); } else { syncActiveChatSessionThinkingState(); } @@ -34462,6 +34670,32 @@ async function handleSendPrompt(options = {}) { // is enabled, so clicking Speak later never needs to read raw markdown. const presenterMode = true; + // The server queue record is the authoritative admission/correlation + // carrier. Await its creation before generate; if persistence is + // unavailable the server can still create a direct compatibility row. + if (!request.promptQueueRecordId && request.promptQueueRecordPromise) { + const activeRecord = await request.promptQueueRecordPromise; + request.promptQueueRecordId = activeRecord?.queueId || null; + } + + // Queue persistence can yield before the server turn starts. Honour an + // abort in that window and remove only this still-queued record so a + // later tab refresh cannot replay a prompt the user already stopped. + if (request.aborted || !isLiveChatRequest(request)) { + if (request.promptQueueRecordId) { + try { + await deletePersistedChatPromptQueueEntry({ + queueId: request.promptQueueRecordId + }); + request.promptQueueRecordId = null; + publishChatPromptQueueChange('queued_prompt_cancelled'); + } catch (cancelError) { + console.warn('[chatTab] Unable to cancel pre-dispatch queued prompt:', cancelError); + } + } + return; + } + // Start the generate request before polling progress so the first // /von/progress read is less likely to outrun request initialisation. const responsePromise = fetch('/von/generate', { @@ -34473,7 +34707,12 @@ async function handleSendPrompt(options = {}) { body: JSON.stringify({ prompt: promptText, client_request_id: request.clientRequestId, + attempt_id: request.attemptId, conversation_session_id: targetSessionId, + conversation_session_name: targetSessionName, + ...(request.promptQueueRecordId ? { + prompt_queue_id: request.promptQueueRecordId + } : {}), ...(request.turnKind ? { turn_kind: request.turnKind } : {}), ...(request.initiationId ? { initiation_id: request.initiationId } : {}), user_id: userContext.user_id, @@ -34493,6 +34732,7 @@ async function handleSendPrompt(options = {}) { startToolUseProgressPolling(request); startForegroundTaskResultPolling(request, { onCompleted: async (generateBody) => { + request.serverQueueTerminalObserved = true; request.attachmentBindingAccepted = true; const delivered = deliverSuccessfulResponseData(generateBody, 'task_result'); if (delivered) { @@ -34504,6 +34744,7 @@ async function handleSendPrompt(options = {}) { } }, onFailed: async (statusPayload, status) => { + request.serverQueueTerminalObserved = true; const delivered = status === 'cancelled' ? deliverCancelledResponseData(statusPayload) : deliverErrorResponseData( @@ -34560,8 +34801,97 @@ async function handleSendPrompt(options = {}) { } if (response.ok) { + request.serverQueueTerminalObserved = true; deliverSuccessfulResponseData(data, 'generate_response'); + } else if ( + data?.retryable === true + || response.status === 429 + ) { + // Admission backpressure is not a failed user task. Restore the + // durable row to queued state and retry after the server's hint; + // another conversation can continue independently meanwhile. + request.admissionDeferred = true; + request.foregroundDeliveryCompleted = true; + request.turnOutcome = { + status: CHAT_PROMPT_QUEUE_STATUS_QUEUED, + phase_label: 'Queued', + summary: data?.detail || 'Waiting for this conversation or server capacity.' + }; + const responseQueueId = (typeof data?.prompt_queue_id === 'string' && data.prompt_queue_id.trim()) + ? data.prompt_queue_id.trim() + : null; + if (!request.promptQueueRecordId && responseQueueId) { + request.promptQueueRecordId = responseQueueId; + } + if (data?.error === 'window_context_unavailable') { + try { + await repairCurrentWindowOrganisationBinding(); + } catch (repairError) { + console.warn('[chatTab] Unable to rebind this tab organisation:', repairError); + } + } + try { + if (request.promptQueueRecordId) { + const refreshed = await refreshChatPromptQueueFromServer({ + silent: true, + skipDrain: true + }); + const canonicalEntry = refreshed + ? refreshed.items.find((entry) => ( + entry?.queueId === request.promptQueueRecordId + )) + : null; + const recovered = canonicalEntry?.status === CHAT_PROMPT_QUEUE_STATUS_QUEUED; + if (canonicalEntry) { + // This exact row can be hidden while the retiring live + // request still owns the active conversation UI. Keep + // it locally so it can drain or render after teardown. + mergePersistedChatPromptQueueEntry(canonicalEntry); + } + if ( + canonicalEntry + && canonicalEntry.status !== CHAT_PROMPT_QUEUE_STATUS_QUEUED + ) { + // Another tab/server attempt already owns this exact + // record. Preserve canonical ownership and wait for a + // queue-change signal instead of locally downgrading it. + request.admissionRetrySuppressed = true; + } else if (!recovered) { + mergePersistedChatPromptQueueEntry({ + queue_id: request.promptQueueRecordId, + prompt_raw: request.promptRaw, + session_id: request.sessionId, + session_name: request.sessionName, + client_request_id: request.clientRequestId, + attempt_id: request.attemptId, + status: CHAT_PROMPT_QUEUE_STATUS_QUEUED + }); + } + } else { + const requeued = await queuePromptForLater(request.promptRaw, { + sessionId: request.sessionId, + sessionName: request.sessionName, + clientRequestId: request.clientRequestId, + attemptId: request.attemptId + }); + request.promptQueueRecordId = requeued?.queueId || null; + } + } catch (requeueError) { + console.info('[chatTab] Admission-deferred row recovery:', requeueError); + } + const retryAfterSeconds = Number.parseInt( + response.headers?.get?.('Retry-After') || data?.retry_after_seconds || '1', + 10 + ); + request.admissionRetryAfterMs = Math.max( + 250, + (Number.isFinite(retryAfterSeconds) ? retryAfterSeconds : 1) * 1000 + ); + if (isRequestVisible()) { + showToast('Prompt queued until this conversation is free.', 'info'); + } } else { + request.serverQueueTerminalObserved = true; deliverErrorResponseData(data, 'An error occurred'); } } catch (error) { @@ -34596,29 +34926,9 @@ async function handleSendPrompt(options = {}) { // The promise already logs failures; proceed without restart recovery. } } - if (request?.promptQueueRecordId) { - const outcomeStatus = canonicalThinkingTerminalStatus(request.turnOutcome?.status); - const terminalStatus = request.aborted || outcomeStatus === THINKING_STATUS_CANCELLED - ? CHAT_PROMPT_QUEUE_STATUS_CANCELLED - : ( - outcomeStatus === THINKING_STATUS_FAILED || outcomeStatus === THINKING_STATUS_TERMINATED - ? CHAT_PROMPT_QUEUE_STATUS_FAILED - : CHAT_PROMPT_QUEUE_STATUS_COMPLETED - ); - const terminalError = terminalStatus === CHAT_PROMPT_QUEUE_STATUS_FAILED - ? (request.turnOutcome?.summary || 'Prompt failed') - : null; - try { - await finishPersistedChatPromptQueueRecord( - request.promptQueueRecordId, - terminalStatus, - terminalError - ); - } catch (error) { - console.warn('[chatTab] Unable to finish persisted prompt queue record:', error); - void refreshChatPromptQueueFromServer({ silent: true }); - } - } + // `/von/generate` now terminalises the exact bound queue attempt. + // Client-side inference is not authoritative, especially after a tab + // switch, aborted response stream, or background reconciliation. if (isLiveChatRequest(request)) { stopToolUseProgressPolling(request); stopThinkingTooltipTicker(request); @@ -34627,6 +34937,10 @@ async function handleSendPrompt(options = {}) { setFinishedThinkingCardForSession(request.sessionId, null); } setLiveChatRequestForSession(request.sessionId, null); + if (request.admissionRetrySuppressed) { + renderChatTaskQueuePanel(); + refreshChatSessionTabActivityIndicators(); + } if (request.sessionId === activeChatSessionId) { void refreshConversationRuntimeCostSnapshot({ observeRequestId: request.clientRequestId, @@ -34637,7 +34951,17 @@ async function handleSendPrompt(options = {}) { syncActiveChatSessionThinkingState(); updateHistoryLength(); scheduleChatSessionTabsRefresh(true); - scheduleQueuedChatPromptDrain(); + if (request.serverQueueTerminalObserved) { + publishChatPromptQueueChange('turn_terminal'); + } + if (request.admissionDeferred && !request.admissionRetrySuppressed) { + setTimeout( + () => scheduleQueuedChatPromptDrain(), + request.admissionRetryAfterMs || 1000 + ); + } else if (!request.admissionRetrySuppressed) { + scheduleQueuedChatPromptDrain(); + } } } @@ -38239,6 +38563,9 @@ export async function __testOnly_openConversationSearchResult(row = {}) { export async function __testOnly_refreshChatPromptQueueFromServer() { return refreshChatPromptQueueFromServer({ silent: false }); } +export function __testOnly_handleChatPromptQueueStorageEvent(event) { + handleChatPromptQueueStorageEvent(event); +} export function __testOnly_resetChatRequestState() { if (conversationRuntimeCostState.handoverRetryTimerId) { clearTimeout(conversationRuntimeCostState.handoverRetryTimerId); @@ -38277,6 +38604,10 @@ export function __testOnly_resetChatRequestState() { clearTimeout(queuedChatPromptDrainTimer); queuedChatPromptDrainTimer = null; } + queuedChatPromptDrainPromise = null; + queuedChatPromptDrainRequested = false; + queuedChatPromptClaimsInFlight.clear(); + chatPromptQueueAdmissionServerInstanceId = null; for (const timer of queuedChatPromptSyncTimers.values()) { clearTimeout(timer); } diff --git a/src/frontend/web/von_interface/static/js/components/orgSelector.js b/src/frontend/web/von_interface/static/js/components/orgSelector.js index 2381243b9..39e67e4cf 100644 --- a/src/frontend/web/von_interface/static/js/components/orgSelector.js +++ b/src/frontend/web/von_interface/static/js/components/orgSelector.js @@ -15,6 +15,11 @@ import { clearRetryableLoadState, describeRetryableLoadFailure, } from '../utils/retryableLoadState.js'; +import { + getSessionScopedOrgContext, + hasSessionPersonalOrgContext, + setSessionScopedOrgContext, +} from '../utils/sessionScopedStorage.js'; // JVNAUTOSCI-1011: Switch to sessionStorage for window-scoped org context const SS_ORG_CONTEXT = 'von_org_context'; @@ -114,19 +119,17 @@ export async function switchOrganisation(orgConceptId, orgName = null) { // JVNAUTOSCI-1011: Store in sessionStorage for window-scoped context sessionStorage.setItem(SS_ORG_CONTEXT, JSON.stringify(orgData)); sessionStorage.setItem(SS_ORG_ROLE, response.role || 'member'); - sessionStorage.setItem(SS_CURRENT_ORG, JSON.stringify(currentOrgData)); + setSessionScopedOrgContext(currentOrgData); // Also store in localStorage for persistence across restarts/new windows localStorage.setItem(LS_ORG_CONTEXT, JSON.stringify(orgData)); localStorage.setItem(LS_ORG_ROLE, response.role || 'member'); - localStorage.setItem(SS_CURRENT_ORG, JSON.stringify(currentOrgData)); } else { sessionStorage.removeItem(SS_ORG_CONTEXT); sessionStorage.removeItem(SS_ORG_ROLE); - sessionStorage.removeItem(SS_CURRENT_ORG); localStorage.removeItem(LS_ORG_CONTEXT); localStorage.removeItem(LS_ORG_ROLE); - localStorage.removeItem(SS_CURRENT_ORG); + setSessionScopedOrgContext(null); } if (response.namespace) { @@ -259,22 +262,7 @@ export async function renderOrgSelector(containerId) { * falling back to localStorage for backward compatibility */ export function getStoredOrgContext() { - // Try sessionStorage first (window-scoped) - try { - const sessionStored = sessionStorage.getItem(SS_ORG_CONTEXT); - if (sessionStored) { - return JSON.parse(sessionStored); - } - } catch { - // Fall through to localStorage - } - // Fallback to localStorage for backward compatibility - try { - const stored = localStorage.getItem(LS_ORG_CONTEXT); - return stored ? JSON.parse(stored) : null; - } catch { - return null; - } + return getSessionScopedOrgContext(); } /** @@ -282,6 +270,9 @@ export function getStoredOrgContext() { * JVNAUTOSCI-1011: Now reads from sessionStorage first */ export function getStoredOrgRole() { + if (hasSessionPersonalOrgContext()) { + return null; + } // Try sessionStorage first const sessionRole = sessionStorage.getItem(SS_ORG_ROLE); if (sessionRole) { diff --git a/src/frontend/web/von_interface/static/js/components/paperRecommendationProfilePanel.js b/src/frontend/web/von_interface/static/js/components/paperRecommendationProfilePanel.js index b0d46ac24..fbb3ec9cc 100644 --- a/src/frontend/web/von_interface/static/js/components/paperRecommendationProfilePanel.js +++ b/src/frontend/web/von_interface/static/js/components/paperRecommendationProfilePanel.js @@ -1,5 +1,6 @@ import { getJsonDetailed, + ensureUniqueWindowSessionId, getWindowSessionId, WINDOW_SESSION_HEADER, } from '../apiService.js'; @@ -196,6 +197,7 @@ export async function ensureRecommendationProfilePanelForConceptTab({ elements.saveButton.onclick = async () => { setStatus(elements.statusElement, 'Saving recommendation profile...'); try { + await ensureUniqueWindowSessionId?.(); const response = await fetch( `/api/concepts/${encodeURIComponent(conceptId)}/paper_recommendation_profile`, { diff --git a/src/frontend/web/von_interface/static/js/domUtils.js b/src/frontend/web/von_interface/static/js/domUtils.js index 60493331f..bfee3a933 100644 --- a/src/frontend/web/von_interface/static/js/domUtils.js +++ b/src/frontend/web/von_interface/static/js/domUtils.js @@ -1,7 +1,11 @@ import { openSettingsTabAndFocus } from './utils/settingsNavigation.js'; import { parseStoredContextValue } from './utils/runtimeIdentityBootstrap.js'; import { applyLocalModelPreferenceOverlay, getEffectiveLocalModelPreference } from './utils/localModelPreferences.js'; -import { getWindowSessionId, WINDOW_SESSION_HEADER } from './apiService.js'; +import { ensureUniqueWindowSessionId, getWindowSessionId, WINDOW_SESSION_HEADER } from './apiService.js'; +import { + getSessionScopedOrgContext, + hasSessionPersonalOrgContext, +} from './utils/sessionScopedStorage.js'; import { formatConversationRuntimeCostFooter } from './utils/conversationRuntimeCost.js'; import { getLatestWorkflowCapabilityIndexStatus, @@ -142,21 +146,7 @@ export function updateHeaderOrgName() { if (!headerOrgNameEl) return; // Read current org from session-scoped storage - let orgName = null; - try { - const sessionOrg = sessionStorage.getItem('von_current_org'); - if (sessionOrg) { - const parsed = parseStoredContextValue(sessionOrg); - orgName = parsed?.name || null; - } - if (!orgName) { - const localOrg = localStorage.getItem('von_current_org'); - if (localOrg) { - const parsed = parseStoredContextValue(localOrg); - orgName = parsed?.name || null; - } - } - } catch { /* ignore */ } + const orgName = getSessionScopedOrgContext()?.name || null; const displayName = orgName || 'Personal'; headerOrgNameEl.textContent = displayName; @@ -171,21 +161,7 @@ async function loadInfoPopupContent() { if (!contentEl) return; // Get current org concept ID - let orgConceptId = null; - try { - const sessionOrg = sessionStorage.getItem('von_current_org'); - if (sessionOrg) { - const parsed = parseStoredContextValue(sessionOrg); - orgConceptId = parsed?.concept_id || null; - } - if (!orgConceptId) { - const localOrg = localStorage.getItem('von_current_org'); - if (localOrg) { - const parsed = parseStoredContextValue(localOrg); - orgConceptId = parsed?.concept_id || null; - } - } - } catch { /* ignore */ } + const orgConceptId = getSessionScopedOrgContext()?.concept_id || null; // Clear cache when loading new org if (orgConceptId !== _orgDescriptionCacheId) { @@ -271,7 +247,8 @@ export function getCurrentUserConceptId() { return stored?.concept_id || null; } -function buildFooterActorHeaders(extraHeaders = {}) { +async function buildFooterActorHeaders(extraHeaders = {}) { + await ensureUniqueWindowSessionId?.(); const userConceptId = getCurrentUserConceptId(); return { [WINDOW_SESSION_HEADER]: getWindowSessionId(), @@ -285,7 +262,7 @@ async function fetchJsonWithTimeout(url, options = {}) { const { timeoutMs = 6000, ...fetchOptions } = options || {}; const init = { ...fetchOptions, - headers: buildFooterActorHeaders(fetchOptions.headers || {}), + headers: await buildFooterActorHeaders(fetchOptions.headers || {}), }; let timeoutId = null; @@ -336,6 +313,9 @@ async function getSettings() { // Helpers to access current user / organisation with both name and concept id. // JVNAUTOSCI-1011: For window-scoped values, check sessionStorage first (per-window), then localStorage (shared fallback) function readSessionScopedJson(key) { + if (key === 'von_current_org' || key === 'von_org_context') { + return getSessionScopedOrgContext(); + } try { const sessionVal = sessionStorage.getItem(key); if (sessionVal) return parseStoredContextValue(sessionVal); @@ -358,6 +338,9 @@ async function getCurrentOrganisationInfo(settingsOverride = null) { const name = switching.name ? `${switching.name} (switching…)` : 'Switching…'; return { id: null, conceptId: switching.concept_id || null, name }; } + if (hasSessionPersonalOrgContext()) { + return { id: null, conceptId: null, name: null }; + } const stored = readSessionScopedJson('von_current_org'); if (stored) { return { id: stored.id || null, conceptId: stored.concept_id || null, name: stored.name || null }; diff --git a/src/frontend/web/von_interface/static/js/main.js b/src/frontend/web/von_interface/static/js/main.js index a2abc886d..37a7c5914 100644 --- a/src/frontend/web/von_interface/static/js/main.js +++ b/src/frontend/web/von_interface/static/js/main.js @@ -20,7 +20,9 @@ import { } from './utils/copyJsonButtonState.js'; import { getSessionScopedNamespace, + getSessionScopedOrgContext, hasSessionOrgContext, + hasSessionPersonalOrgContext, setSessionScopedNamespace, setSessionScopedOrgContext, syncNamespaceFromLocalStorage, @@ -55,9 +57,14 @@ document.addEventListener('DOMContentLoaded', async () => { // Dynamic positioning: calculate header height and position tabs accordingly setupDynamicLayout(); + // Resolve duplicate-tab identity and bind this tab's explicit organisation + // (including Personal) before any child frame or actor-scoped bootstrap read. + await initialiseWindowActorContext(); + // Ensure user context is loaded BEFORE initializing chat to prevent race condition // where conversation history loads with null user_id await ensureUserContext(); + loadDeferredSettingsFrame(); // JVNAUTOSCI-954: report bounded, client-reported capability hints (speech/audio) import('./clientCapabilitiesReporter.js').then(module => { @@ -186,6 +193,34 @@ document.addEventListener('DOMContentLoaded', async () => { console.log("Initial data loads complete."); }); +function loadDeferredSettingsFrame() { + const settingsFrame = document.getElementById('settingsFrame'); + if (!(settingsFrame instanceof HTMLIFrameElement)) return; + const deferredSrc = String(settingsFrame.dataset.src || '').trim(); + if (!deferredSrc || settingsFrame.dataset.actorContextReady === 'true') return; + settingsFrame.dataset.actorContextReady = 'true'; + settingsFrame.src = deferredSrc; +} + +async function initialiseWindowActorContext() { + try { + initSessionStorageFromLocalStorage(); + const { ensureUniqueWindowSessionId } = await import('./apiService.js'); + await ensureUniqueWindowSessionId(); + await syncFlaskSessionOrg(); + } catch (error) { + // Keep the outer page usable, but do not eagerly load the Settings frame + // until the bounded identity attempt has completed. + console.warn('[main] Failed to initialise the window actor context:', error); + } +} + +window.addEventListener('von:windowSessionIdentityChanged', () => { + // A late same-origin collision response can rotate the provisional ID after + // startup. Rebind the retained per-tab organisation to the replacement ID. + void syncFlaskSessionOrg(); +}); + function applyExpertTabGuards() { if (isExpertTabsEnabled()) return; @@ -299,9 +334,10 @@ function initSessionStorageFromLocalStorage() { async function syncFlaskSessionOrg() { try { // Import to ensure window session ID is generated - const { getJson, postJson, getWindowSessionId } = await import('./apiService.js'); - // Ensure window session ID exists - getWindowSessionId(); + const { getJson, postJson, ensureUniqueWindowSessionId } = await import('./apiService.js'); + // Resolve a copied duplicate-tab ID before reading or mutating the + // server-side window organisation context. + await ensureUniqueWindowSessionId(); // First, check if window session already has an org const context = await getJson('/von/api/session/context'); @@ -332,15 +368,18 @@ async function syncFlaskSessionOrg() { // Window session store is empty (or context came from Flask session fallback). // Sync from localStorage to window session store - this is the user's INTENDED org. - let storedOrg = sessionStorage.getItem('von_current_org'); - if (!storedOrg) { - // Fallback to localStorage for backward compatibility - storedOrg = localStorage.getItem('von_current_org'); - } + const storedOrg = getSessionScopedOrgContext(); if (!storedOrg) { console.log('[main] No org in sessionStorage/localStorage, proceeding without org'); - // If Flask session had an org but storage doesn't, clear window session to match - if (context.organisation_id && !isWindowSessionSource) { + // Personal is an explicit per-tab binding. Materialise it even when the + // cookie fallback also happens to be null, so later calls cannot drift to + // another tab's organisation through a non-window fallback. + if (!isWindowSessionSource && hasSessionPersonalOrgContext()) { + console.log('[main] Binding explicit Personal context to this window session'); + await postJson('/von/api/session/set_organisation', { + organisation_concept_id: null + }); + } else if (context.organisation_id && !isWindowSessionSource) { console.log('[main] Clearing stale Flask session org - no org in storage'); await postJson('/von/api/session/set_organisation', { organisation_concept_id: null @@ -348,8 +387,7 @@ async function syncFlaskSessionOrg() { } return; } - const org = parseStoredContextValue(storedOrg); - const orgConceptId = org?.concept_id; + const orgConceptId = storedOrg?.concept_id; if (!orgConceptId) { console.log('[main] No org concept_id in storage, proceeding without org'); return; @@ -403,9 +441,9 @@ async function ensureUserContext() { initSessionStorageFromLocalStorage(); // Import to ensure window session ID is generated - const { getJson, getWindowSessionId } = await import('./apiService.js'); - // Ensure window session ID exists - getWindowSessionId(); + const { getJson, ensureUniqueWindowSessionId } = await import('./apiService.js'); + // Resolve a copied duplicate-tab ID before any actor-scoped bootstrap call. + await ensureUniqueWindowSessionId(); const storedUser = parseStoredContextValue(localStorage.getItem('von_current_user')); if (storedUser?.concept_id) { @@ -423,6 +461,11 @@ async function ensureUserContext() { } } + // Preference hydration may have supplied an organisation for an otherwise + // unset tab. Bind that explicit tab selection before settings can observe + // a cookie-global fallback through the newly unique window ID. + await syncFlaskSessionOrg(); + console.log('[main] Fetching settings to populate user context...'); const settings = await getJson('/api/settings/'); const needsIdentityFallback = !settings.current_user_person_concept_id || !settings.current_organisation_concept_id; @@ -446,12 +489,14 @@ async function ensureUserContext() { console.log('[main] Populated von_current_user from browser bootstrap context'); } - const resolvedOrg = resolveBrowserBootstrapOrganisationContext({ - settings, - sessionContext, - storedOrganisation: parseStoredContextValue(sessionStorage.getItem('von_current_org')) - || parseStoredContextValue(localStorage.getItem('von_current_org')) - }); + const personalContextSelected = hasSessionPersonalOrgContext(); + const resolvedOrg = personalContextSelected + ? null + : resolveBrowserBootstrapOrganisationContext({ + settings, + sessionContext, + storedOrganisation: getSessionScopedOrgContext() + }); if (resolvedOrg) { setSessionScopedOrgContext(resolvedOrg); @@ -459,8 +504,15 @@ async function ensureUserContext() { } const resolvedNamespace = resolveBrowserBootstrapNamespace({ - settings, - sessionContext, + settings: personalContextSelected + ? { + ...settings, + current_organisation_id: null, + current_organisation_concept_id: null, + current_organisation_name: null + } + : settings, + sessionContext: personalContextSelected ? null : sessionContext, userContext: resolvedUser, organisationContext: resolvedOrg }); @@ -526,17 +578,17 @@ document.addEventListener('orgSwitched', (event) => { try { if (orgId) { - // Sync org to parent's sessionStorage so footer reads correct value - const existing = sessionStorage.getItem('von_current_org'); - const parsed = existing ? JSON.parse(existing) : {}; - sessionStorage.setItem('von_current_org', JSON.stringify({ + // Sync org to this tab so footer and request context update together. + const parsed = getSessionScopedOrgContext() || {}; + setSessionScopedOrgContext({ id: parsed.id || null, concept_id: orgId, name: orgName || parsed.name || null - })); + }); } else { - // Switching to personal (no org) - clear sessionStorage entry - sessionStorage.removeItem('von_current_org'); + // Personal is an explicit per-tab selection, not permission to fall + // through to another tab's shared localStorage organisation. + setSessionScopedOrgContext(null); } if (namespace) { sessionStorage.setItem('current_user_namespace', namespace); diff --git a/src/frontend/web/von_interface/static/js/settingsPage.js b/src/frontend/web/von_interface/static/js/settingsPage.js index 332841ba2..32316eaba 100644 --- a/src/frontend/web/von_interface/static/js/settingsPage.js +++ b/src/frontend/web/von_interface/static/js/settingsPage.js @@ -1,4 +1,4 @@ -import { getWindowSessionId, postJson, WINDOW_SESSION_HEADER } from './apiService.js'; +import { ensureUniqueWindowSessionId, getWindowSessionId, postJson, WINDOW_SESSION_HEADER } from './apiService.js'; import { clearBackgroundTaskHistory, formatBackgroundTaskSummary, @@ -45,6 +45,8 @@ import { import { getSessionScopedOrgContext, getSessionScopedNamespace, + hasSessionPersonalOrgContext, + setSessionScopedOrgContext, setSessionScopedNamespace, } from './utils/sessionScopedStorage.js'; import { @@ -88,7 +90,8 @@ import { } from './utils/workflowCapabilityStatusCoordinator.js'; // Helper to build fetch headers with window session context (JVNAUTOSCI-1011) -function buildSettingsFetchHeaders(extraHeaders = {}) { +async function buildSettingsFetchHeaders(extraHeaders = {}) { + await ensureUniqueWindowSessionId?.(); return { [WINDOW_SESSION_HEADER]: getWindowSessionId(), ...extraHeaders @@ -1951,7 +1954,7 @@ async function reloadScopedModelSettings({ try { const response = await fetchFn(settingsUrl, { cache: 'no-store', - headers: buildSettingsFetchHeaders(), + headers: await buildSettingsFetchHeaders(), }); if (!response.ok) { throw new Error(`Failed to fetch scoped model settings: ${response.statusText || response.status}`); @@ -2115,7 +2118,7 @@ async function refreshSelectedOpenAiCostSummary() { try { const response = await fetch(url, { cache: 'no-store', - headers: buildSettingsFetchHeaders(), + headers: await buildSettingsFetchHeaders(), }); if (response.ok) summary = await response.json(); } catch (_) { /* unavailable */ } @@ -2927,7 +2930,7 @@ async function _fetchSessionContextForRole() { try { const resp = await fetch('/von/api/session/context', { cache: 'no-cache', - headers: buildSettingsFetchHeaders() + headers: await buildSettingsFetchHeaders() }); if (!resp.ok) return null; return await resp.json(); @@ -2940,7 +2943,7 @@ async function fetchSettingsAuthStatus() { try { const resp = await fetch('/von/api/auth/status', { cache: 'no-cache', - headers: buildSettingsFetchHeaders() + headers: await buildSettingsFetchHeaders() }); if (!resp.ok) return null; return await resp.json(); @@ -4598,6 +4601,9 @@ window.addEventListener('beforeunload', () => { }); function getStoredJson(key) { + if (key === LS_ORG_KEY) { + return getSessionScopedOrgContext(); + } // JVNAUTOSCI-1011: sessionStorage (per-window) first, localStorage fallback try { const sessionVal = sessionStorage.getItem(key); @@ -4606,6 +4612,10 @@ function getStoredJson(key) { } catch { return null; } } function setStoredJson(key, value) { + if (key === LS_ORG_KEY) { + setSessionScopedOrgContext(value); + return; + } // JVNAUTOSCI-1011: Write to both sessionStorage (window-scoped) and localStorage (persistent) try { if (value == null) { @@ -5388,7 +5398,7 @@ async function loadAndDisplaySettings() { const response = await fetch(settingsUrl, { cache: 'no-store', - headers: buildSettingsFetchHeaders(), + headers: await buildSettingsFetchHeaders(), }); if (!response.ok) throw new Error(`Failed to fetch settings: ${response.statusText}`); const settings = await response.json(); @@ -5409,7 +5419,8 @@ async function loadAndDisplaySettings() { const bootstrapOrg = resolveBrowserBootstrapOrganisationContext({ settings, sessionContext, - storedOrganisation: storedOrg + storedOrganisation: storedOrg, + explicitPersonal: hasSessionPersonalOrgContext(), }); if (bootstrapOrg) { setStoredJson(LS_ORG_KEY, bootstrapOrg); @@ -5419,7 +5430,8 @@ async function loadAndDisplaySettings() { settings, sessionContext, userContext: bootstrapUser, - organisationContext: bootstrapOrg + organisationContext: bootstrapOrg, + explicitPersonal: hasSessionPersonalOrgContext(), }); try { if (bootstrapNamespace) { @@ -5587,10 +5599,10 @@ async function loadAndDisplaySettings() { const orgData = { id: null, concept_id: orgId, name: resolvedName }; // Update both storages so footer reads correct value immediately setStoredJson(LS_ORG_KEY, orgData); - try { window.parent?.sessionStorage?.setItem('von_current_org', JSON.stringify(orgData)); } catch { } + setSessionScopedOrgContext(orgData); } else { setStoredJson(LS_ORG_KEY, null); - try { window.parent?.sessionStorage?.removeItem('von_current_org'); } catch { } + setSessionScopedOrgContext(null); } } catch { } if (window.parent?.updateModelInfoFooterDisplay) { window.parent.updateModelInfoFooterDisplay(); } @@ -6087,7 +6099,7 @@ async function saveAllSettings({ const response = await fetch('/api/settings/', { method: 'POST', - headers: buildSettingsFetchHeaders({ 'Content-Type': 'application/json' }), + headers: await buildSettingsFetchHeaders({ 'Content-Type': 'application/json' }), body: JSON.stringify(settings), }); const payload = await response.json().catch(() => ({})); @@ -6417,7 +6429,7 @@ export async function loadSettings() { try { const response = await fetch('/api/settings/', { cache: 'no-store', - headers: buildSettingsFetchHeaders(), + headers: await buildSettingsFetchHeaders(), }); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); @@ -6433,7 +6445,7 @@ export async function saveSettings(settings) { try { const response = await fetch('/api/settings/', { method: 'POST', - headers: buildSettingsFetchHeaders({ + headers: await buildSettingsFetchHeaders({ 'Content-Type': 'application/json' }), body: JSON.stringify(settings) diff --git a/src/frontend/web/von_interface/static/js/test/chatTab.test.js b/src/frontend/web/von_interface/static/js/test/chatTab.test.js index 340c5f573..93edc55a1 100644 --- a/src/frontend/web/von_interface/static/js/test/chatTab.test.js +++ b/src/frontend/web/von_interface/static/js/test/chatTab.test.js @@ -57,6 +57,7 @@ import { __testOnly_copyActiveThinkingDiagnostics, __testOnly_shouldAcceptThinkingProgressUpdate, __testOnly_applyImmediateThinkingTerminalOutcome, + __testOnly_getForegroundTaskResultPollInitialDelayMs, __testOnly_getThinkingProgressPollFetchTimeoutMs, __testOnly_setThinkingCardRequests, __testOnly_setThinkingState, @@ -873,10 +874,9 @@ describe('workflow monitor capability-index warning cartouche', () => { describe('workflow monitor capability-index polling and global furl', () => { async function flushMicrotasks() { - await Promise.resolve(); - await Promise.resolve(); - await Promise.resolve(); - await Promise.resolve(); + for (let count = 0; count < 8; count += 1) { + await Promise.resolve(); + } } beforeEach(() => { @@ -1030,6 +1030,7 @@ describe('workflow monitor capability-index polling and global furl', () => { })); const refreshPromise = __testOnly_refreshWorkflowCapabilityIndexStatus(); + await flushMicrotasks(); jest.advanceTimersByTime(8000); await refreshPromise; @@ -7522,7 +7523,7 @@ describe('thinking card toggle accessibility', () => { terminal_status: 'effect_partially_completed', response: 'Useful completed work, with one known-no-change failure.' }, 'Complete'] - ])('terminalises a %s delivery before queue persistence finishes', async (_label, responseOk, responseData, badgeText) => { + ])('terminalises a %s delivery without client-side queue terminalisation', async (_label, responseOk, responseData, badgeText) => { const { getUserContext } = require('../apiService.js'); getUserContext.mockReturnValue({ user_id: 'user', @@ -7532,16 +7533,10 @@ describe('thinking card toggle accessibility', () => { }); document.getElementById('promptInput').value = 'Complete this bounded turn'; - let resolveFinish; - let markFinishStarted; - const finishResponse = new Promise((resolve) => { - resolveFinish = resolve; - }); - const finishStarted = new Promise((resolve) => { - markFinishStarted = resolve; - }); + const fetchCalls = []; global.fetch = jest.fn((url, options = {}) => { + fetchCalls.push({ url, options }); if (url === '/von/api/chat_prompt_queue' && options.method === 'POST') { return Promise.resolve({ ok: true, @@ -7550,15 +7545,11 @@ describe('thinking card toggle accessibility', () => { queue_id: 'queue-terminal-test', session_id: 'thinking-card-test-session', prompt_raw: 'Complete this bounded turn', - status: 'in_progress' + status: 'queued' } }) }); } - if (typeof url === 'string' && url.endsWith('/queue-terminal-test/finish')) { - markFinishStarted(); - return finishResponse; - } if (typeof url === 'string' && url.startsWith('/api/settings/')) { return Promise.resolve({ ok: true, @@ -7591,18 +7582,9 @@ describe('thinking card toggle accessibility', () => { return Promise.resolve({ ok: true, json: async () => ({}) }); }); - let sendSettled = false; - const sendPromise = sendMessage().finally(() => { - sendSettled = true; - }); - await finishStarted; - await Promise.resolve(); - - expect(sendSettled).toBe(false); + await expect(sendMessage()).resolves.toBeUndefined(); expect(document.getElementById('thinkingCardStatusBadge').textContent).toBe(badgeText); - - resolveFinish({ ok: true, json: async () => ({ success: true }) }); - await expect(sendPromise).resolves.toBeUndefined(); + expect(fetchCalls.some(({ url }) => String(url).includes('/finish'))).toBe(false); }); test('shows concise critic review from completed thinking card on demand', async () => { @@ -7878,6 +7860,9 @@ describe('thinking card toggle accessibility', () => { expect(wrapper.classList.contains('is-collapsed')).toBe(false); expect(detail.getAttribute('aria-hidden')).toBe('false'); + for (let flush = 0; flush < 8; flush += 1) { + await Promise.resolve(); + } document.getElementById('abortButton').click(); await new Promise((r) => setTimeout(r, 0)); @@ -8014,7 +7999,9 @@ describe('thinking card toggle accessibility', () => { }); const sendPromise = sendMessage(); - await Promise.resolve(); + for (let flush = 0; flush < 8; flush += 1) { + await Promise.resolve(); + } const progressPollTimeoutMs = __testOnly_getThinkingProgressPollFetchTimeoutMs(); const indicatorText = document.querySelector('.loading-indicator-text'); @@ -8067,7 +8054,6 @@ describe('thinking card toggle accessibility', () => { language: 'en-NZ', gmail_profile: null }); - document.getElementById('promptInput').value = 'test prompt'; let generateSignal = null; @@ -8399,20 +8385,22 @@ describe('thinking card toggle accessibility', () => { }); test('renders a completed task result while the foreground generate request is still pending', async () => { - const { getUserContext } = require('../apiService.js'); + const { getUserContext, postJson } = require('../apiService.js'); getUserContext.mockReturnValue({ user_id: 'user', org_id: 'org', language: 'en-NZ', gmail_profile: null }); + postJson.mockReset(); + postJson.mockResolvedValue({ success: true }); __testOnly_setActiveChatSession('session-task-result-bridge', 'Task Result Bridge'); document.getElementById('promptInput').value = 'show the completed side-channel answer'; let generateAbortObserved = false; - const taskStatusUrls = []; - const taskResultUrls = []; + const taskStatusCalls = []; + const taskResultCalls = []; global.fetch = jest.fn((url, options = {}) => { if (typeof url === 'string' && url.startsWith('/api/settings/')) { @@ -8437,7 +8425,18 @@ describe('thinking card toggle accessibility', () => { } if (typeof url === 'string' && url.startsWith('/von/api/task/status/')) { - taskStatusUrls.push(url); + taskStatusCalls.push({ url, options }); + if (taskStatusCalls.length === 1) { + return Promise.resolve({ + ok: false, + status: 409, + json: async () => ({ + success: false, + error_code: 'window_context_unavailable', + retryable: true + }) + }); + } return Promise.resolve({ ok: true, status: 200, @@ -8450,7 +8449,7 @@ describe('thinking card toggle accessibility', () => { } if (typeof url === 'string' && url.startsWith('/von/api/task/result/')) { - taskResultUrls.push(url); + taskResultCalls.push({ url, options }); return Promise.resolve({ ok: true, status: 200, @@ -8498,25 +8497,56 @@ describe('thinking card toggle accessibility', () => { return Promise.resolve({ ok: true, status: 200, json: async () => ({}) }); }); - await expect(sendMessage()).resolves.toBeUndefined(); - for (let attempt = 0; attempt < 10; attempt += 1) { - const renderedText = document.querySelector('.message-container.assistant-turn .chat-message-text')?.textContent || ''; - if (renderedText.includes('Rendered from completed task result')) { - break; + jest.useFakeTimers(); + try { + const sendPromise = sendMessage(); + for (let flush = 0; flush < 12; flush += 1) { + await Promise.resolve(); } - await new Promise(resolve => setTimeout(resolve, 10)); - } - const assistantMessages = Array.from(document.querySelectorAll('.message-container.assistant-turn .chat-message-text')); - expect(assistantMessages).toHaveLength(1); - expect(assistantMessages[0].textContent).toContain('Rendered from completed task result'); - expect(taskStatusUrls.length).toBeGreaterThan(0); - expect(taskResultUrls.length).toBeGreaterThan(0); - expect(generateAbortObserved).toBe(true); + const initialDelayMs = __testOnly_getForegroundTaskResultPollInitialDelayMs(); + expect(initialDelayMs).toBeGreaterThanOrEqual(10_000); + expect(taskStatusCalls).toHaveLength(0); - const retained = getRetainedThinkingCardElements(); - expect(retained).not.toBeNull(); - expect(retained.detail.innerHTML).toContain('Response generated from completed task result'); + jest.advanceTimersByTime(initialDelayMs - 1); + await Promise.resolve(); + expect(taskStatusCalls).toHaveLength(0); + + jest.advanceTimersByTime(1); + for (let flush = 0; flush < 20; flush += 1) { + await Promise.resolve(); + } + expect(taskStatusCalls).toHaveLength(1); + expect(taskResultCalls).toHaveLength(0); + expect(postJson).toHaveBeenCalledWith('/von/api/session/set_organisation', { + organisation_concept_id: 'org' + }); + + jest.advanceTimersByTime(249); + await Promise.resolve(); + expect(taskStatusCalls).toHaveLength(1); + + jest.advanceTimersByTime(1); + for (let flush = 0; flush < 20; flush += 1) { + await Promise.resolve(); + } + await expect(sendPromise).resolves.toBeUndefined(); + + const assistantMessages = Array.from(document.querySelectorAll('.message-container.assistant-turn .chat-message-text')); + expect(assistantMessages).toHaveLength(1); + expect(assistantMessages[0].textContent).toContain('Rendered from completed task result'); + expect(taskStatusCalls.length).toBeGreaterThan(0); + expect(taskResultCalls.length).toBeGreaterThan(0); + expect(taskStatusCalls[0].options.credentials).toBe('same-origin'); + expect(taskResultCalls[0].options.credentials).toBe('same-origin'); + expect(generateAbortObserved).toBe(true); + + const retained = getRetainedThinkingCardElements(); + expect(retained).not.toBeNull(); + expect(retained.detail.innerHTML).toContain('Response generated from completed task result'); + } finally { + jest.useRealTimers(); + } }); test('retains a failed card inline when the last live progress remains non-terminal', async () => { @@ -9155,9 +9185,38 @@ describe('chat abort behaviour', () => { // jsdom best-effort } - let generateSignal = null; + const fetchCalls = []; + let resolveQueueCreate = null; global.fetch = jest.fn((url, options = {}) => { + fetchCalls.push({ url, options }); + if (url === '/von/api/chat_prompt_queue' && options.method === 'POST') { + return new Promise((resolve) => { + resolveQueueCreate = () => resolve({ + ok: true, + status: 201, + json: async () => ({ + success: true, + item: { + queue_id: 'queue-aborted-before-dispatch', + prompt_raw: "since we'\n", + session_id: 'chat-abort-test-session', + status: 'queued' + } + }) + }); + }); + } + if ( + url === '/von/api/chat_prompt_queue/queue-aborted-before-dispatch' + && options.method === 'DELETE' + ) { + return Promise.resolve({ + ok: true, + status: 200, + json: async () => ({ success: true }) + }); + } if (typeof url === 'string' && url.startsWith('/von/history/length')) { return Promise.resolve({ ok: true, @@ -9166,16 +9225,7 @@ describe('chat abort behaviour', () => { } if (typeof url === 'string' && url.startsWith('/von/generate')) { - generateSignal = options.signal; - return new Promise((resolve, reject) => { - if (generateSignal) { - generateSignal.addEventListener('abort', () => { - const err = new Error('aborted'); - err.name = 'AbortError'; - reject(err); - }); - } - }); + throw new Error('aborted pre-dispatch prompt must not generate'); } return Promise.resolve({ ok: true, json: async () => ({}) }); @@ -9188,12 +9238,17 @@ describe('chat abort behaviour', () => { expect(document.getElementById('abortButton').getAttribute('aria-hidden')).toBe('false'); document.getElementById('abortButton').click(); + expect(resolveQueueCreate).not.toBeNull(); + resolveQueueCreate(); // Let abort propagate through promise chain await new Promise((r) => setTimeout(r, 0)); - expect(generateSignal).not.toBeNull(); - expect(generateSignal.aborted).toBe(true); + expect(fetchCalls.some(({ url }) => url === '/von/generate')).toBe(false); + expect(fetchCalls.some(({ url, options }) => ( + url === '/von/api/chat_prompt_queue/queue-aborted-before-dispatch' + && options.method === 'DELETE' + ))).toBe(true); expect(document.getElementById('sendButton').disabled).toBe(false); expect(document.getElementById('abortButton').getAttribute('aria-hidden')).toBe('true'); expect(promptInput.value).toBe("since we'\n"); @@ -9540,12 +9595,12 @@ describe('chat session composer state', () => { expect(backgroundTab).not.toBeNull(); expect(backgroundTab.classList.contains('has-background-request')).toBe(true); expect(abortSpy).not.toHaveBeenCalled(); - expect(document.getElementById('sendButton').textContent).toBe('Queue Prompt'); + expect(document.getElementById('sendButton').textContent).toBe('Send Prompt'); expect(promptInput.value).toBe(''); expect(overlayContent.textContent).toBe(''); }); - test('queued prompts stay pinned to their originating session after switching away again', async () => { + test('requests in different sessions overlap and remain pinned to their originating sessions', async () => { const promptInput = document.getElementById('promptInput'); initializePromptCartoucheOverlay(promptInput); @@ -9638,12 +9693,15 @@ describe('chat session composer state', () => { expect.objectContaining({ ok: true }) ); - promptInput.value = 'queued for session 2'; + promptInput.value = 'request for session 2'; await expect(sendMessage()).resolves.toBeUndefined(); - const queueLabels = Array.from(document.querySelectorAll('.chat-task-queue-item-label')) - .map((node) => node.textContent || ''); - expect(queueLabels).toContain('Next up • Target'); + // Session 2 starts before Session 1 resolves; neither conversation + // enters the other's FIFO. + expect(generateBodies).toHaveLength(2); + expect(generateBodies[0].conversation_session_id).toBe('session-1'); + expect(generateBodies[1].conversation_session_id).toBe('session-2'); + expect(document.querySelector('.chat-task-queue-item')).toBeNull(); await expect(switchToChatSession('session-1')).resolves.toEqual( expect.objectContaining({ ok: true }) diff --git a/src/frontend/web/von_interface/static/js/test/sessionScopedStorage.test.js b/src/frontend/web/von_interface/static/js/test/sessionScopedStorage.test.js index ea8f30675..8f4aa4d5e 100644 --- a/src/frontend/web/von_interface/static/js/test/sessionScopedStorage.test.js +++ b/src/frontend/web/von_interface/static/js/test/sessionScopedStorage.test.js @@ -1,6 +1,12 @@ import { buildNamespaceScopedStorageKey, + clearAllOrgContext, + getSessionScopedOrgContext, getSessionScopedNamespace, + hasSessionOrgContext, + hasSessionPersonalOrgContext, + setSessionScopedOrgContext, + syncOrgContextFromLocalStorage, syncNamespaceFromLocalStorage, } from '../utils/sessionScopedStorage.js'; @@ -69,4 +75,53 @@ describe('sessionScopedStorage namespace repair', () => { 'von:openConceptTabs:%23V%23michael_witbrock%40university_of_auckland_strong_ai_lab', ); }); + + test('keeps an explicit Personal tab isolated when another tab changes the shared organisation default', () => { + localStorage.setItem( + 'von_current_user', + JSON.stringify({ concept_id: '#V#michael_witbrock' }), + ); + setSessionScopedOrgContext(null); + sessionStorage.setItem('current_user_namespace', '#V#michael_witbrock'); + + // Model another tab subsequently selecting an organisation through the + // shared persistence compatibility keys. + localStorage.setItem( + 'von_current_org', + JSON.stringify({ concept_id: '#V#organisation_b' }), + ); + localStorage.setItem( + 'current_user_namespace', + '#V#michael_witbrock@organisation_b', + ); + + expect(hasSessionPersonalOrgContext()).toBe(true); + expect(hasSessionOrgContext()).toBe(true); + expect(getSessionScopedOrgContext()).toBeNull(); + expect(getSessionScopedNamespace()).toBe('#V#michael_witbrock'); + }); + + test('selecting an organisation clears Personal and a fresh tab still inherits the shared default', () => { + setSessionScopedOrgContext(null); + expect(hasSessionPersonalOrgContext()).toBe(true); + + setSessionScopedOrgContext({ concept_id: '#V#organisation_a' }); + + expect(hasSessionPersonalOrgContext()).toBe(false); + expect(getSessionScopedOrgContext()).toEqual({ concept_id: '#V#organisation_a' }); + + sessionStorage.clear(); + syncOrgContextFromLocalStorage(); + expect(getSessionScopedOrgContext()).toEqual({ concept_id: '#V#organisation_a' }); + }); + + test('logout clears the explicit Personal marker', () => { + setSessionScopedOrgContext(null); + expect(hasSessionPersonalOrgContext()).toBe(true); + + clearAllOrgContext(); + + expect(hasSessionPersonalOrgContext()).toBe(false); + expect(hasSessionOrgContext()).toBe(false); + }); }); diff --git a/src/frontend/web/von_interface/static/js/test/userPreferenceBootstrap.test.js b/src/frontend/web/von_interface/static/js/test/userPreferenceBootstrap.test.js index 49430899f..65f98fbfa 100644 --- a/src/frontend/web/von_interface/static/js/test/userPreferenceBootstrap.test.js +++ b/src/frontend/web/von_interface/static/js/test/userPreferenceBootstrap.test.js @@ -60,4 +60,20 @@ describe('userPreferenceBootstrap', () => { expect(setStoredOrgContext).not.toHaveBeenCalled(); expect(localStorage.getItem('von_preferred_language')).toBe('en-NZ'); }); + + test('preserves an explicit Personal selection instead of restoring a preferred organisation', async () => { + const setStoredOrgContext = jest.fn(); + + await hydrateStoredSelectionsFromUserPreferences('#V#michael_witbrock', { + getJsonImpl: jest.fn().mockResolvedValue({ + organisation_concept_id: '#V#university_of_auckland_strong_ai_lab', + }), + getStoredOrgContext: () => null, + hasStoredOrgSelection: () => true, + setStoredOrgContext, + localStorageImpl: localStorage, + }); + + expect(setStoredOrgContext).not.toHaveBeenCalled(); + }); }); diff --git a/src/frontend/web/von_interface/static/js/test/windowSessionIdentity.test.js b/src/frontend/web/von_interface/static/js/test/windowSessionIdentity.test.js new file mode 100644 index 000000000..21e74c889 --- /dev/null +++ b/src/frontend/web/von_interface/static/js/test/windowSessionIdentity.test.js @@ -0,0 +1,281 @@ +import { + createWindowSessionIdentityCoordinator, + WINDOW_SESSION_KEY, +} from '../utils/windowSessionIdentity.js'; + +class MemoryStorage { + constructor(seed = {}) { + this.values = new Map(Object.entries(seed)); + } + + getItem(key) { + return this.values.has(key) ? this.values.get(key) : null; + } + + setItem(key, value) { + this.values.set(key, String(value)); + } +} + +function createBroadcastChannelHarness({ delayForMessage = () => 0 } = {}) { + const channelsByName = new Map(); + + return class FakeBroadcastChannel { + constructor(name) { + this.name = name; + this.listeners = new Set(); + this.closed = false; + const channels = channelsByName.get(name) || new Set(); + channels.add(this); + channelsByName.set(name, channels); + } + + addEventListener(type, listener) { + if (type === 'message') this.listeners.add(listener); + } + + removeEventListener(type, listener) { + if (type === 'message') this.listeners.delete(listener); + } + + postMessage(data) { + const delayMs = delayForMessage(data); + const deliver = () => { + for (const peer of channelsByName.get(this.name) || []) { + if (peer === this || peer.closed) continue; + for (const listener of peer.listeners) { + listener({ data }); + } + } + }; + if (delayMs > 0) { + setTimeout(deliver, delayMs); + } else { + queueMicrotask(deliver); + } + } + + close() { + this.closed = true; + channelsByName.get(this.name)?.delete(this); + } + }; +} + +function createDocumentIdFactory(prefix) { + let counter = 0; + return () => `${prefix}-${counter += 1}`; +} + +function createCoordinator({ + BroadcastChannelImpl, + storage, + priority, + replacementId, + collisionProbeMs = 5, + eventTarget = null, + isTopLevelDocument = true, +}) { + return createWindowSessionIdentityCoordinator({ + BroadcastChannelImpl, + storage, + documentPriority: priority, + createDocumentInstanceId: createDocumentIdFactory(`document-${priority}`), + createWindowSessionId: jest.fn(() => replacementId), + collisionProbeMs, + eventTarget, + isTopLevelDocument, + }); +} + +describe('window session identity coordination', () => { + test('rotates only a copied claimant while the established owner retains its ID', async () => { + const BroadcastChannelImpl = createBroadcastChannelHarness(); + const ownerStorage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_copied' }); + const owner = createCoordinator({ + BroadcastChannelImpl, + storage: ownerStorage, + priority: '001', + replacementId: 'ws_owner_unused', + }); + await expect(owner.ensureUniqueWindowSessionId()).resolves.toBe('ws_copied'); + + const claimantStorage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_copied' }); + const claimant = createCoordinator({ + BroadcastChannelImpl, + storage: claimantStorage, + priority: '002', + replacementId: 'ws_claimant_replacement', + }); + + await expect(claimant.ensureUniqueWindowSessionId()).resolves.toBe('ws_claimant_replacement'); + expect(ownerStorage.getItem(WINDOW_SESSION_KEY)).toBe('ws_copied'); + expect(claimantStorage.getItem(WINDOW_SESSION_KEY)).toBe('ws_claimant_replacement'); + + owner.close(); + claimant.close(); + }); + + test('uses probe priority so simultaneous claimants do not both rotate', async () => { + const BroadcastChannelImpl = createBroadcastChannelHarness(); + const earlierStorage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_shared' }); + const laterStorage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_shared' }); + const earlier = createCoordinator({ + BroadcastChannelImpl, + storage: earlierStorage, + priority: '001', + replacementId: 'ws_earlier_unused', + }); + const later = createCoordinator({ + BroadcastChannelImpl, + storage: laterStorage, + priority: '002', + replacementId: 'ws_later_replacement', + }); + + await expect(Promise.all([ + earlier.ensureUniqueWindowSessionId(), + later.ensureUniqueWindowSessionId(), + ])).resolves.toEqual(['ws_shared', 'ws_later_replacement']); + expect(earlierStorage.getItem(WINDOW_SESSION_KEY)).toBe('ws_shared'); + expect(laterStorage.getItem(WINDOW_SESSION_KEY)).toBe('ws_later_replacement'); + + earlier.close(); + later.close(); + }); + + test('preserves a reload ID when the retiring document has released its channel', async () => { + const BroadcastChannelImpl = createBroadcastChannelHarness(); + const storage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_reload' }); + const beforeReload = createCoordinator({ + BroadcastChannelImpl, + storage, + priority: '001', + replacementId: 'ws_before_reload_unused', + }); + await beforeReload.ensureUniqueWindowSessionId(); + beforeReload.close(); + + const afterReload = createCoordinator({ + BroadcastChannelImpl, + storage, + priority: '002', + replacementId: 'ws_after_reload_unused', + }); + + await expect(afterReload.ensureUniqueWindowSessionId()).resolves.toBe('ws_reload'); + expect(storage.getItem(WINDOW_SESSION_KEY)).toBe('ws_reload'); + afterReload.close(); + }); + + test('does not let a same-origin iframe claim its top-level tab ID', async () => { + let constructedChannels = 0; + const BaseChannel = createBroadcastChannelHarness(); + class CountingChannel extends BaseChannel { + constructor(name) { + super(name); + constructedChannels += 1; + } + } + const storage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_parent' }); + const iframe = createCoordinator({ + BroadcastChannelImpl: CountingChannel, + storage, + priority: '002', + replacementId: 'ws_iframe_must_not_use', + isTopLevelDocument: false, + }); + + await expect(iframe.ensureUniqueWindowSessionId()).resolves.toBe('ws_parent'); + expect(constructedChannels).toBe(0); + expect(storage.getItem(WINDOW_SESSION_KEY)).toBe('ws_parent'); + }); + + test('rotates and notifies after a delayed incumbent response', async () => { + const BroadcastChannelImpl = createBroadcastChannelHarness({ + delayForMessage: (message) => message?.type === 'occupied' ? 20 : 0, + }); + const ownerStorage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_delayed' }); + const owner = createCoordinator({ + BroadcastChannelImpl, + storage: ownerStorage, + priority: '001', + replacementId: 'ws_owner_unused', + }); + await owner.ensureUniqueWindowSessionId(); + + const eventTarget = { + CustomEvent, + addEventListener: jest.fn(), + dispatchEvent: jest.fn(), + }; + const claimantStorage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_delayed' }); + const claimant = createCoordinator({ + BroadcastChannelImpl, + storage: claimantStorage, + priority: '002', + replacementId: 'ws_late_replacement', + eventTarget, + }); + + await expect(claimant.ensureUniqueWindowSessionId()).resolves.toBe('ws_delayed'); + await new Promise((resolve) => setTimeout(resolve, 60)); + + expect(claimantStorage.getItem(WINDOW_SESSION_KEY)).toBe('ws_late_replacement'); + expect(eventTarget.dispatchEvent).toHaveBeenCalledWith(expect.objectContaining({ + type: 'von:windowSessionIdentityChanged', + })); + + owner.close(); + claimant.close(); + }); + + test('cancels a hidden-page probe and notifies when BFCache restore must rotate', async () => { + const BroadcastChannelImpl = createBroadcastChannelHarness(); + const eventListeners = new Map(); + const eventTarget = { + CustomEvent, + addEventListener: jest.fn((type, listener) => { + eventListeners.set(type, listener); + }), + dispatchEvent: jest.fn(), + }; + const restoredStorage = new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_bfcache' }); + const restored = createCoordinator({ + BroadcastChannelImpl, + storage: restoredStorage, + priority: '002', + replacementId: 'ws_bfcache_replacement', + collisionProbeMs: 20, + eventTarget, + }); + + const hiddenProbe = restored.ensureUniqueWindowSessionId(); + eventListeners.get('pagehide')?.({}); + await expect(hiddenProbe).resolves.toBe('ws_bfcache'); + + const owner = createCoordinator({ + BroadcastChannelImpl, + storage: new MemoryStorage({ [WINDOW_SESSION_KEY]: 'ws_bfcache' }), + priority: '001', + replacementId: 'ws_owner_unused', + collisionProbeMs: 20, + }); + await owner.ensureUniqueWindowSessionId(); + + eventListeners.get('pageshow')?.({ persisted: true }); + await new Promise((resolve) => setTimeout(resolve, 70)); + + expect(restoredStorage.getItem(WINDOW_SESSION_KEY)).toBe('ws_bfcache_replacement'); + expect(eventTarget.dispatchEvent).toHaveBeenCalledWith(expect.objectContaining({ + type: 'von:windowSessionIdentityChanged', + detail: { + previous_window_session_id: 'ws_bfcache', + window_session_id: 'ws_bfcache_replacement', + }, + })); + + owner.close(); + restored.close(); + }); +}); diff --git a/src/frontend/web/von_interface/static/js/utils/governedPublicationScope.js b/src/frontend/web/von_interface/static/js/utils/governedPublicationScope.js index f3117e83e..f2dc9a2b0 100644 --- a/src/frontend/web/von_interface/static/js/utils/governedPublicationScope.js +++ b/src/frontend/web/von_interface/static/js/utils/governedPublicationScope.js @@ -1,4 +1,4 @@ -import { getWindowSessionId, WINDOW_SESSION_HEADER } from '../apiService.js'; +import { ensureUniqueWindowSessionId, getWindowSessionId, WINDOW_SESSION_HEADER } from '../apiService.js'; const CHANGE_REASON = 'concept_tab_publication_scope_change'; const USER_PREDICATE = '#V#specific_to_user'; @@ -239,7 +239,8 @@ function requestId(phase) { return `concept-scope-${phase}-${token}`; } -function headers(json = false) { +async function headers(json = false) { + await ensureUniqueWindowSessionId?.(); return { ...(json ? { 'Content-Type': 'application/json' } : {}), [WINDOW_SESSION_HEADER]: getWindowSessionId() @@ -361,7 +362,7 @@ export async function executeGovernedScopeControlChange({ const endpoint = `/api/ontology-authority/concepts/${encodeURIComponent(conceptId)}/scope`; const read = await responseJson(await fetchImpl(endpoint, { credentials: 'same-origin', - headers: headers() + headers: await headers() }), 'Could not read the current publication scope.'); const fingerprint = String(read.scope_fingerprint || '').trim(); if (!fingerprint) { @@ -380,7 +381,7 @@ export async function executeGovernedScopeControlChange({ const preview = await responseJson(await fetchImpl(endpoint, { method: 'POST', credentials: 'same-origin', - headers: headers(true), + headers: await headers(true), body: JSON.stringify({ scope_edit: requestedScopeEdit, expected_scope_fingerprint: fingerprint, @@ -402,7 +403,7 @@ export async function executeGovernedScopeControlChange({ const result = await responseJson(await fetchImpl(endpoint, { method: 'POST', credentials: 'same-origin', - headers: headers(true), + headers: await headers(true), body: JSON.stringify({ scope_edit: validated.resolvedScopeEdit, expected_scope_fingerprint: fingerprint, diff --git a/src/frontend/web/von_interface/static/js/utils/runtimeIdentityBootstrap.js b/src/frontend/web/von_interface/static/js/utils/runtimeIdentityBootstrap.js index db691051c..2f5e23d13 100644 --- a/src/frontend/web/von_interface/static/js/utils/runtimeIdentityBootstrap.js +++ b/src/frontend/web/von_interface/static/js/utils/runtimeIdentityBootstrap.js @@ -82,7 +82,11 @@ export function resolveBrowserBootstrapOrganisationContext({ settings = null, sessionContext = null, storedOrganisation = null, + explicitPersonal = false, } = {}) { + if (explicitPersonal) { + return null; + } const conceptId = trimString( settings?.current_organisation_concept_id || sessionContext?.organisation_id @@ -110,14 +114,17 @@ export function resolveBrowserBootstrapNamespace({ sessionContext = null, userContext = null, organisationContext = null, + explicitPersonal = false, } = {}) { const sessionNamespace = trimString(sessionContext?.namespace); - if (sessionNamespace) { + if (sessionNamespace && (!explicitPersonal || !sessionNamespace.includes('@'))) { return sessionNamespace; } return buildNamespaceFromConcepts( settings?.current_user_person_concept_id || userContext?.concept_id || null, - settings?.current_organisation_concept_id || organisationContext?.concept_id || null + explicitPersonal + ? null + : (settings?.current_organisation_concept_id || organisationContext?.concept_id || null) ); } diff --git a/src/frontend/web/von_interface/static/js/utils/sessionScopedStorage.js b/src/frontend/web/von_interface/static/js/utils/sessionScopedStorage.js index 316a379e1..3d4ee0c72 100644 --- a/src/frontend/web/von_interface/static/js/utils/sessionScopedStorage.js +++ b/src/frontend/web/von_interface/static/js/utils/sessionScopedStorage.js @@ -16,8 +16,20 @@ const KEYS = { NAMESPACE: 'current_user_namespace', NAMESPACE_LEGACY: 'von_namespace', ORG_SWITCHING: 'von_org_switching', + ORG_SELECTION: 'von_org_selection', }; +const ORG_SELECTION_PERSONAL = 'personal'; +const ORG_SELECTION_ORGANISATION = 'organisation'; + +export function hasSessionPersonalOrgContext() { + try { + return sessionStorage.getItem(KEYS.ORG_SELECTION) === ORG_SELECTION_PERSONAL; + } catch { + return false; + } +} + function readJsonFromStorage(storage, key) { try { const raw = storage?.getItem(key); @@ -63,11 +75,14 @@ export function deriveNamespaceFromStoredContext() { const userSlug = conceptIdToNamespaceSlug(storedUser?.concept_id); if (!userSlug) return ''; - const storedOrg = - readJsonFromStorage(session, KEYS.CURRENT_ORG) - || readJsonFromStorage(local, KEYS.CURRENT_ORG) - || readJsonFromStorage(session, KEYS.ORG_CONTEXT) - || readJsonFromStorage(local, KEYS.ORG_CONTEXT); + const storedOrg = hasSessionPersonalOrgContext() + ? null + : ( + readJsonFromStorage(session, KEYS.CURRENT_ORG) + || readJsonFromStorage(local, KEYS.CURRENT_ORG) + || readJsonFromStorage(session, KEYS.ORG_CONTEXT) + || readJsonFromStorage(local, KEYS.ORG_CONTEXT) + ); const orgSlug = conceptIdToNamespaceSlug(storedOrg?.concept_id); return orgSlug ? `#V#${userSlug}@${orgSlug}` : `#V#${userSlug}`; @@ -81,6 +96,17 @@ function resolvePreferredNamespaceFromStorage() { const sessionLegacyNamespace = readTrimmedStorageValue(session, KEYS.NAMESPACE_LEGACY); const localLegacyNamespace = readTrimmedStorageValue(local, KEYS.NAMESPACE_LEGACY); const derivedNamespace = deriveNamespaceFromStoredContext(); + if (hasSessionPersonalOrgContext()) { + // An explicit Personal selection is a per-tab value, not absence of a + // value. Never inherit another tab's organisation-scoped namespace + // from shared localStorage. + if (derivedNamespace) { + return derivedNamespace; + } + return [sessionNamespace, sessionLegacyNamespace] + .find((candidate) => candidate && !isOrgScopedNamespace(candidate)) + || ''; + } const storedCandidates = [ sessionNamespace, localNamespace, @@ -133,6 +159,10 @@ function repairPrimaryNamespaceStorage(namespace) { * @returns {Object|null} Parsed org object with id, concept_id, name, namespace, or null */ export function getSessionScopedOrgContext() { + if (hasSessionPersonalOrgContext()) { + return null; + } + // Try von_current_org from sessionStorage first try { const sessionOrg = sessionStorage.getItem(KEYS.CURRENT_ORG); @@ -206,10 +236,17 @@ export function buildNamespaceScopedStorageKey(prefix, namespace = null) { export function setSessionScopedOrgContext(orgData) { try { if (orgData == null) { + // Preserve Personal as an explicit per-tab choice. Without this + // tombstone, a later organisation selection in another tab would + // leak back through the shared localStorage compatibility path. + sessionStorage.setItem(KEYS.ORG_SELECTION, ORG_SELECTION_PERSONAL); sessionStorage.removeItem(KEYS.CURRENT_ORG); + sessionStorage.removeItem(KEYS.ORG_CONTEXT); localStorage.removeItem(KEYS.CURRENT_ORG); + localStorage.removeItem(KEYS.ORG_CONTEXT); } else { const json = JSON.stringify(orgData); + sessionStorage.setItem(KEYS.ORG_SELECTION, ORG_SELECTION_ORGANISATION); sessionStorage.setItem(KEYS.CURRENT_ORG, json); localStorage.setItem(KEYS.CURRENT_ORG, json); } @@ -238,7 +275,8 @@ export function setSessionScopedNamespace(namespace) { */ export function hasSessionOrgContext() { try { - return !!(sessionStorage.getItem(KEYS.CURRENT_ORG) || sessionStorage.getItem(KEYS.ORG_CONTEXT)); + return hasSessionPersonalOrgContext() + || !!(sessionStorage.getItem(KEYS.CURRENT_ORG) || sessionStorage.getItem(KEYS.ORG_CONTEXT)); } catch { return false; } @@ -265,12 +303,18 @@ export function syncOrgContextFromLocalStorage() { try { const lsOrg = localStorage.getItem(KEYS.CURRENT_ORG); - if (lsOrg) sessionStorage.setItem(KEYS.CURRENT_ORG, lsOrg); + if (lsOrg) { + sessionStorage.setItem(KEYS.CURRENT_ORG, lsOrg); + sessionStorage.setItem(KEYS.ORG_SELECTION, ORG_SELECTION_ORGANISATION); + } } catch { /* ignore */ } try { const lsCtx = localStorage.getItem(KEYS.ORG_CONTEXT); - if (lsCtx) sessionStorage.setItem(KEYS.ORG_CONTEXT, lsCtx); + if (lsCtx) { + sessionStorage.setItem(KEYS.ORG_CONTEXT, lsCtx); + sessionStorage.setItem(KEYS.ORG_SELECTION, ORG_SELECTION_ORGANISATION); + } } catch { /* ignore */ } } @@ -292,6 +336,7 @@ export function clearAllOrgContext() { sessionStorage.removeItem(KEYS.NAMESPACE); sessionStorage.removeItem(KEYS.NAMESPACE_LEGACY); sessionStorage.removeItem(KEYS.ORG_SWITCHING); + sessionStorage.removeItem(KEYS.ORG_SELECTION); localStorage.removeItem(KEYS.CURRENT_ORG); localStorage.removeItem(KEYS.ORG_CONTEXT); localStorage.removeItem(KEYS.NAMESPACE); @@ -301,4 +346,3 @@ export function clearAllOrgContext() { // Export keys for direct access if needed (e.g., for specific checks) export { KEYS as STORAGE_KEYS }; - diff --git a/src/frontend/web/von_interface/static/js/utils/userPreferenceBootstrap.js b/src/frontend/web/von_interface/static/js/utils/userPreferenceBootstrap.js index feb7fe57a..425c9c95a 100644 --- a/src/frontend/web/von_interface/static/js/utils/userPreferenceBootstrap.js +++ b/src/frontend/web/von_interface/static/js/utils/userPreferenceBootstrap.js @@ -1,6 +1,7 @@ import { getJson } from '../apiService.js'; import { getSessionScopedOrgContext, + hasSessionOrgContext, setSessionScopedOrgContext, } from './sessionScopedStorage.js'; @@ -36,6 +37,7 @@ function buildStoredOrgContext(organisationConceptId, existingOrgContext = null) export async function hydrateStoredSelectionsFromUserPreferences(userConceptId, { getJsonImpl = getJson, getStoredOrgContext = getSessionScopedOrgContext, + hasStoredOrgSelection = hasSessionOrgContext, setStoredOrgContext = setSessionScopedOrgContext, localStorageImpl = typeof localStorage !== 'undefined' ? localStorage : null, } = {}) { @@ -46,7 +48,13 @@ export async function hydrateStoredSelectionsFromUserPreferences(userConceptId, ? getStoredOrgContext() : null; - if (!existingOrgContext?.concept_id && prefs.organisation_concept_id) { + const hasExplicitOrgSelection = Boolean(existingOrgContext?.concept_id) + || ( + typeof hasStoredOrgSelection === 'function' + && hasStoredOrgSelection() + ); + + if (!hasExplicitOrgSelection && prefs.organisation_concept_id) { const orgContext = buildStoredOrgContext( prefs.organisation_concept_id, existingOrgContext, diff --git a/src/frontend/web/von_interface/static/js/utils/windowSessionIdentity.js b/src/frontend/web/von_interface/static/js/utils/windowSessionIdentity.js new file mode 100644 index 000000000..3c819e95d --- /dev/null +++ b/src/frontend/web/von_interface/static/js/utils/windowSessionIdentity.js @@ -0,0 +1,389 @@ +const DEFAULT_WINDOW_SESSION_KEY = 'von_window_session_id'; +const DEFAULT_CHANNEL_NAME = 'von_window_session_coordination_v1'; +const DEFAULT_COLLISION_PROBE_MS = 75; + +function createOpaqueId(prefix, cryptoImpl, nowImpl, randomImpl) { + const randomId = ( + cryptoImpl + && typeof cryptoImpl.randomUUID === 'function' + ) + ? cryptoImpl.randomUUID().replace(/-/g, '') + : `${nowImpl().toString(36)}${randomImpl().toString(36).slice(2, 12)}`; + return `${prefix}${randomId}`; +} + +/** + * Coordinate the opaque ID that binds one browser tab to one server-side + * window context. Browsers copy sessionStorage into a duplicated/opener-created + * tab, so sessionStorage alone does not guarantee uniqueness. + * + * The newly loading document probes same-origin tabs over BroadcastChannel. + * An existing owner only answers; the probing document rotates its copied ID. + * Reloads retain the tab ID even if the retiring document briefly answers. + */ +export function createWindowSessionIdentityCoordinator(options = {}) { + const storage = options.storage + ?? (typeof sessionStorage !== 'undefined' ? sessionStorage : null); + const BroadcastChannelImpl = options.BroadcastChannelImpl + ?? ( + typeof window !== 'undefined' + && typeof window.BroadcastChannel === 'function' + ? window.BroadcastChannel + : null + ); + const performanceImpl = options.performanceImpl + ?? (typeof performance !== 'undefined' ? performance : null); + const eventTarget = options.eventTarget + ?? (typeof window !== 'undefined' ? window : null); + const cryptoImpl = options.cryptoImpl + ?? (typeof crypto !== 'undefined' ? crypto : null); + const nowImpl = options.nowImpl || Date.now; + const randomImpl = options.randomImpl || Math.random; + const setTimeoutImpl = options.setTimeoutImpl || setTimeout; + const clearTimeoutImpl = options.clearTimeoutImpl || clearTimeout; + const storageKey = options.storageKey || DEFAULT_WINDOW_SESSION_KEY; + const channelName = options.channelName || DEFAULT_CHANNEL_NAME; + const collisionProbeMs = Number.isFinite(Number(options.collisionProbeMs)) + ? Math.max(0, Number(options.collisionProbeMs)) + : DEFAULT_COLLISION_PROBE_MS; + const createWindowSessionId = options.createWindowSessionId + || (() => createOpaqueId('ws_', cryptoImpl, nowImpl, randomImpl)); + const createDocumentInstanceId = options.createDocumentInstanceId + || (() => createOpaqueId('document_', cryptoImpl, nowImpl, randomImpl)); + const isTopLevelDocument = options.isTopLevelDocument + ?? (() => { + if (typeof window === 'undefined') return true; + try { + return window.top === window; + } catch { + return false; + } + })(); + + const documentInstanceId = createDocumentInstanceId(); + const documentPriority = String( + options.documentPriority + ?? `${Number(performanceImpl?.timeOrigin || nowImpl())}:${documentInstanceId}` + ); + let inMemorySessionId = ''; + let channel = null; + let ensurePromise = null; + let pendingProbe = null; + let lastSettledProbe = null; + let established = false; + let lateRotationPromise = null; + let pageHideListenerBound = false; + + const dispatchIdentityChanged = (previousSessionId, settledSessionId) => { + if (!previousSessionId || !settledSessionId || previousSessionId === settledSessionId) { + return; + } + try { + const EventConstructor = eventTarget?.CustomEvent || globalThis.CustomEvent; + if (eventTarget?.dispatchEvent && typeof EventConstructor === 'function') { + eventTarget.dispatchEvent(new EventConstructor( + 'von:windowSessionIdentityChanged', + { + detail: { + previous_window_session_id: previousSessionId, + window_session_id: settledSessionId, + } + } + )); + } + } catch { + // The next actor-scoped request still uses the settled ID. + } + }; + + const getOrCreateWindowSessionId = () => { + let sessionId = inMemorySessionId; + try { + sessionId = String(storage?.getItem(storageKey) || sessionId || '').trim(); + } catch { + sessionId = ''; + } + if (sessionId) { + inMemorySessionId = sessionId; + return sessionId; + } + + sessionId = createWindowSessionId(); + inMemorySessionId = sessionId; + try { + storage?.setItem(storageKey, sessionId); + } catch { + // Keep the in-memory value available for this request even if the + // browser has disabled storage. + } + return sessionId; + }; + + const postChannelMessage = (message) => { + try { + channel?.postMessage(message); + } catch { + // BroadcastChannel is only a collision-recovery enhancement. A + // storage or channel failure must not prevent ordinary requests. + } + }; + + const sendOccupied = (message, currentSessionId) => { + postChannelMessage({ + protocol: 'von_window_session_coordination.v1', + type: 'occupied', + window_session_id: currentSessionId, + source_document_id: documentInstanceId, + source_priority: documentPriority, + target_document_id: message.source_document_id, + probe_id: message.probe_id, + }); + }; + + const rotateLateCollision = () => { + if (lateRotationPromise || !established) { + return; + } + lateRotationPromise = (async () => { + const previousSessionId = getOrCreateWindowSessionId(); + let replacementSessionId = createWindowSessionId(); + while (replacementSessionId === previousSessionId) { + replacementSessionId = createWindowSessionId(); + } + inMemorySessionId = replacementSessionId; + try { + storage?.setItem(storageKey, replacementSessionId); + } catch { + // Keep the in-memory replacement for this document. + } + established = false; + ensurePromise = null; + const settledSessionId = await ensureUniqueWindowSessionId(); + dispatchIdentityChanged(previousSessionId, settledSessionId); + })().finally(() => { + lateRotationPromise = null; + }); + }; + + const handleChannelMessage = (event) => { + const message = event?.data; + if (!message || message.protocol !== 'von_window_session_coordination.v1') { + return; + } + if (message.source_document_id === documentInstanceId) { + return; + } + + const currentSessionId = getOrCreateWindowSessionId(); + if ( + message.type === 'probe' + && message.window_session_id === currentSessionId + ) { + if (established) { + sendOccupied(message, currentSessionId); + } else if (pendingProbe) { + const peerPriority = String(message.source_priority || message.source_document_id || ''); + if (documentPriority > peerPriority) { + pendingProbe.collisionDetected = true; + } else { + sendOccupied(message, currentSessionId); + } + } + return; + } + + if ( + message.type === 'occupied' + && pendingProbe + && message.target_document_id === documentInstanceId + && message.probe_id === pendingProbe.probeId + && message.window_session_id === pendingProbe.windowSessionId + ) { + pendingProbe.collisionDetected = true; + return; + } + + if ( + message.type === 'occupied' + && established + && lastSettledProbe + && message.target_document_id === documentInstanceId + && message.probe_id === lastSettledProbe.probeId + && message.window_session_id === currentSessionId + ) { + rotateLateCollision(); + return; + } + + if ( + message.type === 'claimed' + && message.window_session_id === currentSessionId + ) { + const peerPriority = String(message.source_priority || message.source_document_id || ''); + if (pendingProbe) { + if (documentPriority > peerPriority) { + pendingProbe.collisionDetected = true; + } else { + sendOccupied(message, currentSessionId); + } + } else if (established) { + if (documentPriority > peerPriority) { + rotateLateCollision(); + } else { + sendOccupied(message, currentSessionId); + } + } + } + }; + + const ensureChannel = () => { + if (channel || !isTopLevelDocument || typeof BroadcastChannelImpl !== 'function') { + return channel; + } + try { + channel = new BroadcastChannelImpl(channelName); + channel.addEventListener?.('message', handleChannelMessage); + if (!channel.addEventListener) { + channel.onmessage = handleChannelMessage; + } + if (!pageHideListenerBound && eventTarget?.addEventListener) { + pageHideListenerBound = true; + eventTarget.addEventListener('pagehide', () => { + pendingProbe?.settle?.({ cancelled: true }); + try { + channel?.close?.(); + } catch { + // Ignore teardown races. + } + channel = null; + established = false; + ensurePromise = null; + }); + eventTarget.addEventListener('pageshow', (event) => { + if (event?.persisted) { + ensurePromise = null; + established = false; + void ensureUniqueWindowSessionId(); + } + }); + } + } catch { + channel = null; + } + return channel; + }; + + const probeWindowSessionId = (candidateSessionId) => new Promise((resolve) => { + const probeId = createDocumentInstanceId(); + let settled = false; + let timerId = null; + const probe = { + probeId, + windowSessionId: candidateSessionId, + collisionDetected: false, + settle: ({ cancelled = false } = {}) => { + if (settled) return; + settled = true; + if (timerId !== null) { + clearTimeoutImpl(timerId); + } + const collisionDetected = !cancelled + && probe.collisionDetected === true; + if (pendingProbe?.probeId === probeId) { + pendingProbe = null; + } + if (!cancelled) { + lastSettledProbe = { + probeId, + windowSessionId: candidateSessionId, + }; + } + resolve({ collisionDetected, cancelled }); + }, + }; + pendingProbe = probe; + timerId = setTimeoutImpl(() => probe.settle(), collisionProbeMs); + postChannelMessage({ + protocol: 'von_window_session_coordination.v1', + type: 'probe', + window_session_id: candidateSessionId, + source_document_id: documentInstanceId, + source_priority: documentPriority, + probe_id: probeId, + }); + }); + + const ensureUniqueWindowSessionId = () => { + if (ensurePromise) { + return ensurePromise; + } + + const candidateSessionId = getOrCreateWindowSessionId(); + if (!ensureChannel()) { + ensurePromise = Promise.resolve(candidateSessionId); + return ensurePromise; + } + + ensurePromise = (async () => { + let settledSessionId = candidateSessionId; + for (let probeAttempt = 0; probeAttempt < 3; probeAttempt += 1) { + const { collisionDetected, cancelled } = await probeWindowSessionId(settledSessionId); + if (cancelled) { + return getOrCreateWindowSessionId(); + } + if (!collisionDetected) { + established = true; + postChannelMessage({ + protocol: 'von_window_session_coordination.v1', + type: 'claimed', + window_session_id: settledSessionId, + source_document_id: documentInstanceId, + source_priority: documentPriority, + probe_id: lastSettledProbe?.probeId || null, + }); + dispatchIdentityChanged(candidateSessionId, settledSessionId); + return settledSessionId; + } + + const collidedSessionId = settledSessionId; + do { + settledSessionId = createWindowSessionId(); + } while (settledSessionId === collidedSessionId); + inMemorySessionId = settledSessionId; + try { + storage?.setItem(storageKey, settledSessionId); + } catch { + // Keep the in-memory replacement for this document. + } + } + established = true; + dispatchIdentityChanged(candidateSessionId, settledSessionId); + return settledSessionId; + })(); + return ensurePromise; + }; + + const close = () => { + pendingProbe?.settle?.({ cancelled: true }); + try { + channel?.removeEventListener?.('message', handleChannelMessage); + channel?.close?.(); + } catch { + // Ignore teardown races. + } + channel = null; + established = false; + ensurePromise = null; + }; + + return { + close, + ensureUniqueWindowSessionId, + getWindowSessionId: getOrCreateWindowSessionId, + }; +} + +export { + DEFAULT_CHANNEL_NAME as WINDOW_SESSION_COORDINATION_CHANNEL, + DEFAULT_WINDOW_SESSION_KEY as WINDOW_SESSION_KEY, +}; diff --git a/src/frontend/web/von_interface/static/js/utils/workflowCapabilityStatusCoordinator.js b/src/frontend/web/von_interface/static/js/utils/workflowCapabilityStatusCoordinator.js index ebb7b6cad..6f2be0691 100644 --- a/src/frontend/web/von_interface/static/js/utils/workflowCapabilityStatusCoordinator.js +++ b/src/frontend/web/von_interface/static/js/utils/workflowCapabilityStatusCoordinator.js @@ -1,4 +1,4 @@ -import { getWindowSessionId, WINDOW_SESSION_HEADER } from '../apiService.js'; +import { ensureUniqueWindowSessionId, getWindowSessionId, WINDOW_SESSION_HEADER } from '../apiService.js'; import { parseStoredContextValue } from './runtimeIdentityBootstrap.js'; const COORDINATOR_STATE_KEY = '__vonWorkflowCapabilityIndexStatusCoordinator'; @@ -35,9 +35,10 @@ function readWindowSessionId() { function readCoordinatorNamespace(coordinatorWindow) { try { + const personalSelected = coordinatorWindow?.sessionStorage?.getItem('von_org_selection') === 'personal'; return String( coordinatorWindow?.sessionStorage?.getItem('current_user_namespace') - || coordinatorWindow?.localStorage?.getItem('current_user_namespace') + || (!personalSelected && coordinatorWindow?.localStorage?.getItem('current_user_namespace')) || '', ).trim(); } catch { @@ -51,10 +52,19 @@ function resolveCoordinatorContext(coordinatorWindow) { readStoredContext(coordinatorWindow?.sessionStorage, 'von_current_user') || readStoredContext(coordinatorWindow?.localStorage, 'von_current_user') ); - const organisation = ( - readStoredContext(coordinatorWindow?.sessionStorage, 'von_current_org') - || readStoredContext(coordinatorWindow?.localStorage, 'von_current_org') - ); + const personalSelected = (() => { + try { + return coordinatorWindow?.sessionStorage?.getItem('von_org_selection') === 'personal'; + } catch { + return false; + } + })(); + const organisation = personalSelected + ? null + : ( + readStoredContext(coordinatorWindow?.sessionStorage, 'von_current_org') + || readStoredContext(coordinatorWindow?.localStorage, 'von_current_org') + ); const userConceptId = String(user?.concept_id || user?.conceptId || '').trim(); const organisationConceptId = String( organisation?.concept_id || organisation?.conceptId || '', @@ -177,6 +187,7 @@ export async function refreshWorkflowCapabilityIndexStatus({ fetchImpl = globalThis.fetch, timeoutMs = DEFAULT_STATUS_REFRESH_TIMEOUT_MS, } = {}) { + await ensureUniqueWindowSessionId?.(); const state = getCoordinatorState(); const requestContextKey = state.contextKey; const requestGeneration = state.generation; diff --git a/src/frontend/web/von_interface/static/js/vontology.js b/src/frontend/web/von_interface/static/js/vontology.js index 30a1b08ea..21cff3bb6 100644 --- a/src/frontend/web/von_interface/static/js/vontology.js +++ b/src/frontend/web/von_interface/static/js/vontology.js @@ -1,7 +1,7 @@ import { fetchConceptList, resetConceptTab, updateConceptTabUI } from './conceptTab.js'; import { clearContainer, elements, getCurrentUserConceptId } from './domUtils.js'; import { handleVontologyNodeSelection } from './dynamicTabs.js'; -import { createConcept, getWindowSessionId, WINDOW_SESSION_HEADER } from './apiService.js'; +import { createConcept, ensureUniqueWindowSessionId, getWindowSessionId, WINDOW_SESSION_HEADER } from './apiService.js'; import { createPredicateBadge, getPredicateType } from './predicateUtils.js'; import { ProgressManager, ProgressPhase } from './progress.js'; import { @@ -49,7 +49,8 @@ function debugLog(...args) { } } -function vontologyFetch(url, options = {}) { +async function vontologyFetch(url, options = {}) { + await ensureUniqueWindowSessionId?.(); // Keep identity context consistent across Vontology endpoints. // Access control can fall back to X-User-Concept-ID when session state is absent. const userConceptId = getCurrentUserConceptId(); diff --git a/src/frontend/web/von_interface/static/js/workflowStudioPage.js b/src/frontend/web/von_interface/static/js/workflowStudioPage.js index 8db9ada47..1dad06c14 100644 --- a/src/frontend/web/von_interface/static/js/workflowStudioPage.js +++ b/src/frontend/web/von_interface/static/js/workflowStudioPage.js @@ -1,4 +1,4 @@ -import { getWindowSessionId, WINDOW_SESSION_HEADER } from './apiService.js'; +import { ensureUniqueWindowSessionId, getWindowSessionId, WINDOW_SESSION_HEADER } from './apiService.js'; import { syncNamespaceFromLocalStorage, syncOrgContextFromLocalStorage @@ -345,6 +345,7 @@ function buildWorkflowStudioRequestHeaders(options = {}) { } async function fetchJson(url, options = {}) { + await ensureUniqueWindowSessionId?.(); const { headers: _ignoredHeaders, ...fetchOptions } = options || {}; const response = await fetch(url, { ...fetchOptions, @@ -1532,6 +1533,7 @@ async function initialiseWorkflowStudio() { // Keep the standalone studio page aligned with the main Von window-scoped context model. syncOrgContextFromLocalStorage(); syncNamespaceFromLocalStorage(); + await ensureUniqueWindowSessionId?.(); cacheElements(); bindEvents(); renderAll(); diff --git a/src/frontend/web/von_interface/templates/von_interface.html b/src/frontend/web/von_interface/templates/von_interface.html index 7c4ddea7a..4429c83c4 100644 --- a/src/frontend/web/von_interface/templates/von_interface.html +++ b/src/frontend/web/von_interface/templates/von_interface.html @@ -159,7 +159,7 @@

-
diff --git a/src/workflows/von/main.py b/src/workflows/von/main.py index 8b00eb497..ce998acb4 100644 --- a/src/workflows/von/main.py +++ b/src/workflows/von/main.py @@ -19,15 +19,27 @@ if src_root not in sys.path: sys.path.insert(0, src_root) +_runtime_env = importlib.import_module("src.backend.utils.runtime_env") +apply_repo_dotenv_overrides = _runtime_env.apply_repo_dotenv_overrides + +# The Flask route import constructs the process-wide foreground admission +# service. Load its capacity inputs first so direct main.py launches preserve +# the same HTTP-thread headroom as run.sh launches, which already load .env. +apply_repo_dotenv_overrides( + { + "VON_WAITRESS_THREADS", + "VON_TURN_HTTP_HEADROOM_THREADS", + "VON_MAX_ACTIVE_TURNS_GLOBAL", + "VON_MAX_ACTIVE_TURNS_PER_USER", + } +) + _utils_flask = importlib.import_module("src.backend.server.utils_flask") create_flask_app = _utils_flask.create_flask_app build_browser_entry_url = _utils_flask.build_browser_entry_url get_expert_tabs_enabled = importlib.import_module( "src.backend.services.feature_flags" ).get_expert_tabs_enabled -apply_repo_dotenv_overrides = importlib.import_module( - "src.backend.utils.runtime_env" -).apply_repo_dotenv_overrides # Add src directory to Python path FIRST, before any backend imports @@ -150,6 +162,11 @@ def _apply_dotenv_overrides(keys: set[str]) -> None: # deferral, hot-poller caching, and chat-workflow warm. The launcher and # VS Code hosts do not reliably inherit them, so resolve from .env. "VON_WAITRESS_THREADS", + "VON_TURN_HTTP_HEADROOM_THREADS", + "VON_MAX_ACTIVE_TURNS_GLOBAL", + "VON_MAX_ACTIVE_TURNS_PER_USER", + "VON_MAX_QUEUED_TURNS_GLOBAL", + "VON_MAX_QUEUED_TURNS_PER_USER", "VON_DURABLE_WORKER_POLL_INTERVAL", "VON_DURABLE_WORKER_PRIORITY_RESERVED_SLOTS", "VON_DURABLE_WORKER_PAUSE_BACKGROUND_UNDER_LIVE_LOAD", diff --git a/tests/backend/test_background_task_service.py b/tests/backend/test_background_task_service.py index 2e05a598f..e8b0eccd0 100644 --- a/tests/backend/test_background_task_service.py +++ b/tests/backend/test_background_task_service.py @@ -6,8 +6,8 @@ from __future__ import annotations -import time import threading +import time from datetime import datetime, timezone from typing import Any, Mapping @@ -15,7 +15,9 @@ from src.backend.integrations.internal_mcp.orchestrator import CancellationRequested from src.backend.services.background_task_service import ( + BackgroundTaskCapacityReached, BackgroundTaskRegistry, + BackgroundTaskScopeMismatch, TaskStatus, ) @@ -33,6 +35,13 @@ def test_to_dict_includes_all_fields(self) -> None: started_at=now, progress={"phase": "tool_execute"}, progress_history=[{"phase": "tool_execute"}], + session_id="session-123", + user_id="user-123", + organisation_id="organisation-123", + namespace="#V#user-123@organisation-123", + queue_id="queue-123", + client_request_id="request-123", + attempt_id="attempt-123", ) result = status.to_dict() @@ -47,6 +56,13 @@ def test_to_dict_includes_all_fields(self) -> None: assert result["progress"] == {"phase": "tool_execute"} assert result["progress_history"] == [{"phase": "tool_execute"}] assert result["cancellation_requested"] is False + assert result["session_id"] == "session-123" + assert result["user_id"] == "user-123" + assert result["organisation_id"] == "organisation-123" + assert result["namespace"] == "#V#user-123@organisation-123" + assert result["queue_id"] == "queue-123" + assert result["client_request_id"] == "request-123" + assert result["attempt_id"] == "attempt-123" def test_to_dict_with_completed_task(self) -> None: """to_dict() should handle completed tasks with results.""" @@ -199,15 +215,21 @@ def _failing_task() -> None: registry.shutdown(wait=True) - def test_request_cancellation_sets_flag(self) -> None: - """request_cancellation() should terminally mark the task cancelled.""" + def test_request_cancellation_stays_nonterminal_until_worker_acknowledges( + self, + ) -> None: + """Running work is cancelling, not cancelled, until it actually stops.""" registry = BackgroundTaskRegistry(max_workers=1) + started = threading.Event() + release = threading.Event() def _slow_task() -> str: - time.sleep(10) + started.set() + release.wait(timeout=2) return "done" registry.submit_task(task_id="task-cancel", callable=_slow_task) + assert started.wait(timeout=2) # Request cancellation success = registry.request_cancellation("task-cancel") @@ -219,14 +241,15 @@ def _slow_task() -> str: status = registry.get_task_status("task-cancel") assert status is not None assert status.cancellation_requested is True - assert status.status == "cancelled" - assert status.completed_at is not None - assert status.progress.get("status") == "cancelled" + assert status.status == "running" + assert status.completed_at is None + assert status.progress.get("status") == "cancelling" - registry.shutdown(wait=False) + release.set() + registry.shutdown(wait=True) - def test_request_cancellation_resists_late_worker_completion(self) -> None: - """A cancelled task should not be overwritten by a late worker result.""" + def test_noncooperative_worker_reports_its_actual_late_completion(self) -> None: + """A cancellation request is not a false terminal cancellation receipt.""" registry = BackgroundTaskRegistry(max_workers=1) started = threading.Event() release = threading.Event() @@ -251,12 +274,72 @@ def _late_completing_task() -> str: status = registry.get_task_status("late-cancel") assert status is not None + assert status.status == "completed" + assert status.result == "done" + assert status.error is None + assert status.cancellation_requested is True + + registry.shutdown(wait=True) + + def test_exception_after_cancellation_request_acknowledges_cancellation( + self, + ) -> None: + """A cancel-before-bind race is cancelled rather than failed.""" + + registry = BackgroundTaskRegistry(max_workers=1) + started = threading.Event() + release = threading.Event() + + def _admission_race() -> None: + started.set() + release.wait(timeout=2) + raise RuntimeError("queue row was cancelled before turn admission") + + registry.submit_task(task_id="cancel-before-bind", callable=_admission_race) + assert started.wait(timeout=2) + assert registry.request_cancellation("cancel-before-bind") is True + + release.set() + for _ in range(50): + status = registry.get_task_status("cancel-before-bind") + if status and status.completed_at is not None: + break + time.sleep(0.05) + + status = registry.get_task_status("cancel-before-bind") + assert status is not None assert status.status == "cancelled" - assert status.result is None - assert status.error == "Cancellation requested for task late-cancel" + assert status.cancellation_requested is True + assert status.completed_at is not None + assert status.progress.get("status") == "cancelled" + assert status.error == "queue row was cancelled before turn admission" registry.shutdown(wait=True) + def test_pending_future_can_acknowledge_cancellation_immediately(self) -> None: + registry = BackgroundTaskRegistry(max_workers=1, max_pending_tasks=2) + blocker_started = threading.Event() + release_blocker = threading.Event() + + def _blocker() -> str: + blocker_started.set() + release_blocker.wait(timeout=2) + return "blocker done" + + registry.submit_task(task_id="blocker", callable=_blocker) + assert blocker_started.wait(timeout=2) + registry.submit_task(task_id="pending-cancel", callable=lambda: "must not run") + + assert registry.request_cancellation("pending-cancel") is True + status = registry.get_task_status("pending-cancel") + assert status is not None + assert status.status == "cancelled" + assert status.completed_at is not None + assert status.progress.get("status") == "cancelled" + + release_blocker.set() + registry.shutdown(wait=True) + def test_request_cancellation_returns_false_for_completed(self) -> None: """request_cancellation() should return False for completed tasks.""" registry = BackgroundTaskRegistry(max_workers=1) @@ -342,6 +425,107 @@ def _slow_task() -> str: registry.shutdown(wait=False) + def test_submit_cannot_overwrite_terminal_task_scope(self) -> None: + """A replayed client request ID cannot replace a terminal task record.""" + registry = BackgroundTaskRegistry(max_workers=1) + registry.mark_terminal_external( + "terminal-id", + status="completed", + result={"answer": "first"}, + user_id="#V#first_user", + organisation_id="#V#first_org", + namespace="#V#first_user@first_org", + ) + + with pytest.raises(ValueError, match="already exists"): + registry.submit_task( + task_id="terminal-id", + callable=lambda: "second", + user_id="#V#second_user", + organisation_id="#V#second_org", + namespace="#V#second_user@second_org", + ) + + retained = registry.get_task_status("terminal-id") + assert retained is not None + assert retained.user_id == "#V#first_user" + assert retained.result == {"answer": "first"} + registry.shutdown(wait=True) + + def test_background_capacity_is_bounded_and_fair_across_users(self) -> None: + registry = BackgroundTaskRegistry( + max_workers=4, + max_pending_tasks=2, + max_active_tasks_per_user=2, + ) + release = threading.Event() + + def _blocking_task() -> str: + release.wait(timeout=2) + return "done" + + registry.submit_task( + task_id="user-a-org-1", + callable=_blocking_task, + user_id="#V#user_a", + organisation_id="#V#org_1", + namespace="#V#user_a@org_1", + ) + registry.submit_task( + task_id="user-a-org-2", + callable=_blocking_task, + user_id="#V#user_a", + organisation_id="#V#org_2", + namespace="#V#user_a@org_2", + ) + + with pytest.raises(BackgroundTaskCapacityReached): + registry.submit_task( + task_id="user-a-third", + callable=_blocking_task, + user_id="#V#user_a", + organisation_id="#V#org_3", + namespace="#V#user_a@org_3", + ) + + other = registry.submit_task( + task_id="user-b-first", + callable=_blocking_task, + user_id="#V#user_b", + organisation_id="#V#org_1", + namespace="#V#user_b@org_1", + ) + assert other.user_id == "#V#user_b" + + release.set() + registry.shutdown(wait=True) + + def test_background_pending_queue_has_a_global_bound(self) -> None: + registry = BackgroundTaskRegistry( + max_workers=1, + max_pending_tasks=1, + max_active_tasks_per_user=1, + ) + release = threading.Event() + + registry.submit_task( + task_id="global-running", + callable=lambda: release.wait(timeout=2), + ) + registry.submit_task( + task_id="global-pending", + callable=lambda: release.wait(timeout=2), + ) + + with pytest.raises(BackgroundTaskCapacityReached): + registry.submit_task( + task_id="global-overflow", + callable=lambda: None, + ) + + release.set() + registry.shutdown(wait=True) + def test_remove_task(self) -> None: """remove_task() should remove a task from the registry.""" registry = BackgroundTaskRegistry(max_workers=1) @@ -499,8 +683,8 @@ def _cooperative_task() -> str: registry.shutdown(wait=True) - def test_submit_task_with_session_and_user_id(self) -> None: - """submit_task() should store session_id and user_id (Phase 4).""" + def test_submit_task_with_actor_scope(self) -> None: + """submit_task() should retain conversation and actor scope metadata.""" registry = BackgroundTaskRegistry(max_workers=1) def _task() -> str: @@ -511,6 +695,11 @@ def _task() -> str: callable=_task, session_id="sess-123", user_id="user-456", + organisation_id="organisation-789", + namespace="#V#user-456@organisation-789", + queue_id="queue-456", + client_request_id="request-456", + attempt_id="attempt-456", ) # Wait for completion @@ -524,11 +713,21 @@ def _task() -> str: assert status is not None assert status.session_id == "sess-123" assert status.user_id == "user-456" + assert status.organisation_id == "organisation-789" + assert status.namespace == "#V#user-456@organisation-789" + assert status.queue_id == "queue-456" + assert status.client_request_id == "request-456" + assert status.attempt_id == "attempt-456" # Check to_dict includes the fields status_dict = status.to_dict() assert status_dict["session_id"] == "sess-123" assert status_dict["user_id"] == "user-456" + assert status_dict["organisation_id"] == "organisation-789" + assert status_dict["namespace"] == "#V#user-456@organisation-789" + assert status_dict["queue_id"] == "queue-456" + assert status_dict["client_request_id"] == "request-456" + assert status_dict["attempt_id"] == "attempt-456" registry.shutdown(wait=True) @@ -543,6 +742,8 @@ def test_mark_terminal_external_creates_completed_status(self) -> None: progress={"source": "durable_conversation_turn_instance"}, session_id="session-123", user_id="user-456", + organisation_id="organisation-789", + namespace="#V#user-456@organisation-789", ) assert status.status == "completed" @@ -555,6 +756,8 @@ def test_mark_terminal_external_creates_completed_status(self) -> None: assert restored is not None assert restored.session_id == "session-123" assert restored.user_id == "user-456" + assert restored.organisation_id == "organisation-789" + assert restored.namespace == "#V#user-456@organisation-789" registry.shutdown(wait=True) @@ -595,6 +798,49 @@ def _late_failing_task() -> str: registry.shutdown(wait=True) + def test_mark_terminal_external_rejects_conflicting_actor_scope(self) -> None: + """Terminal reconciliation cannot cross scope bound at submission.""" + registry = BackgroundTaskRegistry(max_workers=1) + started = threading.Event() + release = threading.Event() + + def _task() -> str: + started.set() + release.wait(timeout=2) + return "done" + + registry.submit_task( + task_id="scope-bound", + callable=_task, + session_id="session-original", + user_id="user-original", + organisation_id="organisation-original", + namespace="#V#user-original@organisation-original", + ) + assert started.wait(timeout=2) + + with pytest.raises(BackgroundTaskScopeMismatch): + registry.mark_terminal_external( + "scope-bound", + status="completed", + result={"source": "durable"}, + session_id="session-late", + user_id="user-late", + organisation_id="organisation-late", + namespace="#V#user-late@organisation-late", + ) + + status = registry.get_task_status("scope-bound") + assert status is not None + assert status.status == "running" + assert status.session_id == "session-original" + assert status.user_id == "user-original" + assert status.organisation_id == "organisation-original" + assert status.namespace == "#V#user-original@organisation-original" + + release.set() + registry.shutdown(wait=True) + def test_mark_terminal_external_completed_cannot_override_failed_status( self, ) -> None: @@ -642,7 +888,7 @@ def test_mark_terminal_external_completed_cannot_replace_completed_result( result={"source": "final_payload", "llm_debug": {"request_id": "req-1"}}, progress={"source": "final_generate_payload"}, session_id="session-final", - user_id="user-final", + user_id="user-early", ) assert unchanged.status == "completed" @@ -748,3 +994,108 @@ def _task() -> str: assert user_2_tasks[0]["task_id"] == "user-2-task" registry.shutdown(wait=True) + + def test_list_tasks_filters_by_organisation_and_namespace(self) -> None: + """Organisation and namespace filters should isolate actor scopes.""" + registry = BackgroundTaskRegistry(max_workers=2) + + def _task() -> str: + return "done" + + registry.submit_task( + task_id="org-a-task", + callable=_task, + user_id="same-user", + organisation_id="organisation-A", + namespace="#V#same-user@organisation-A", + ) + registry.submit_task( + task_id="org-b-task", + callable=_task, + user_id="same-user", + organisation_id="organisation-B", + namespace="#V#same-user@organisation-B", + ) + + time.sleep(0.1) + + organisation_a_tasks = registry.list_tasks( + user_id="same-user", + organisation_id="organisation-A", + ) + namespace_b_tasks = registry.list_tasks( + user_id="same-user", + namespace="#V#same-user@organisation-B", + ) + + assert [task["task_id"] for task in organisation_a_tasks] == ["org-a-task"] + assert [task["task_id"] for task in namespace_b_tasks] == ["org-b-task"] + + registry.shutdown(wait=True) + + def test_exact_scope_controls_status_and_cancellation(self) -> None: + """A guessed task ID must not cross user or organisation scope.""" + registry = BackgroundTaskRegistry(max_workers=1) + started = threading.Event() + release = threading.Event() + + def _task() -> str: + started.set() + release.wait(timeout=2) + return "done" + + registry.submit_task( + task_id="scoped-task", + callable=_task, + user_id="#V#same_user", + organisation_id="#V#org_a", + namespace="#V#same_user@org_a", + ) + assert started.wait(timeout=2) + + assert ( + registry.get_task_status_for_scope( + "scoped-task", + user_id="#V#same_user", + organisation_id="#V#org_b", + namespace="#V#same_user@org_b", + ) + is None + ) + assert ( + registry.list_tasks_for_scope( + user_id="#V#same_user", + organisation_id="#V#org_b", + namespace="#V#same_user@org_b", + ) + == [] + ) + assert [ + task["task_id"] + for task in registry.list_tasks_for_scope( + user_id="#V#same_user", + organisation_id="#V#org_a", + namespace="#V#same_user@org_a", + ) + ] == ["scoped-task"] + assert not registry.request_cancellation_for_scope( + "scoped-task", + user_id="#V#other_user", + organisation_id="#V#org_a", + namespace="#V#other_user@org_a", + ) + assert registry.get_task_status_for_scope( + "scoped-task", + user_id="#V#same_user", + organisation_id="#V#org_a", + namespace="#V#same_user@org_a", + ) + assert registry.request_cancellation_for_scope( + "scoped-task", + user_id="#V#same_user", + organisation_id="#V#org_a", + namespace="#V#same_user@org_a", + ) + + release.set() + registry.shutdown(wait=True) diff --git a/tests/backend/test_chat_prompt_queue_routes.py b/tests/backend/test_chat_prompt_queue_routes.py index 0cfd4dc8f..4065ade6a 100644 --- a/tests/backend/test_chat_prompt_queue_routes.py +++ b/tests/backend/test_chat_prompt_queue_routes.py @@ -1,10 +1,12 @@ from __future__ import annotations -from flask import Flask import pytest +from flask import Flask from src.backend.db import mongo_client from src.backend.server.routes.von_routes import von_bp +from src.backend.services import chat_prompt_queue_service +from src.backend.services.conversation_turn_admission_service import SERVER_INSTANCE_ID @pytest.fixture() @@ -56,8 +58,17 @@ def test_chat_prompt_queue_routes_restore_and_complete_record(client) -> None: assert update_resp.get_json()["item"]["prompt_raw"] == "Edited queued task" claim_resp = client.post(f"/von/api/chat_prompt_queue/{queue_id}/claim") - assert claim_resp.status_code == 200 - assert claim_resp.get_json()["item"]["status"] == "in_progress" + assert claim_resp.status_code == 410 + assert claim_resp.get_json()["error_code"] == "queue_claim_route_retired" + + chat_prompt_queue_service.claim_queue_record( + scope=chat_prompt_queue_service.build_queue_scope( + user_concept_id="#V#test_user", + organisation_concept_id="#V#test_org", + namespace="#V#test_user@test_org", + ), + queue_id=queue_id, + ) restartable_resp = client.get("/von/api/chat_prompt_queue") assert restartable_resp.status_code == 200 @@ -78,6 +89,60 @@ def test_chat_prompt_queue_routes_restore_and_complete_record(client) -> None: assert final_list_resp.get_json()["items"] == [] +def test_chat_prompt_queue_create_returns_typed_backpressure( + client, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("VON_MAX_QUEUED_TURNS_GLOBAL", "10") + monkeypatch.setenv("VON_MAX_QUEUED_TURNS_PER_USER", "1") + first = client.post( + "/von/api/chat_prompt_queue", + json={"prompt_raw": "Accepted", "session_id": "session-1"}, + ) + rejected = client.post( + "/von/api/chat_prompt_queue", + json={"prompt_raw": "Rejected", "session_id": "session-2"}, + ) + + assert first.status_code == 201 + assert first.get_json()["item"]["conversation_key"] + assert rejected.status_code == 429 + assert rejected.headers["Retry-After"] == "1" + assert rejected.get_json() == { + "success": False, + "error": "This user has reached the queued-turn backlog limit", + "error_code": "foreground_queue_capacity_reached", + "limit_kind": "user", + "retryable": True, + "retry_after_seconds": 1, + } + + +def test_queue_session_update_recomputes_and_clears_canonical_conversation_key( + client, +) -> None: + created = client.post( + "/von/api/chat_prompt_queue", + json={"prompt_raw": "Move me", "session_id": "session-a"}, + ).get_json()["item"] + + moved = client.patch( + f"/von/api/chat_prompt_queue/{created['queue_id']}", + json={"session_id": "session-b"}, + ) + cleared = client.patch( + f"/von/api/chat_prompt_queue/{created['queue_id']}", + json={"session_id": None}, + ) + + assert moved.status_code == 200 + assert moved.get_json()["item"]["session_id"] == "session-b" + assert moved.get_json()["item"]["conversation_key"] + assert moved.get_json()["item"]["conversation_key"] != created["conversation_key"] + assert cleared.status_code == 200 + assert cleared.get_json()["item"]["session_id"] is None + assert cleared.get_json()["item"]["conversation_key"] is None + + def test_chat_prompt_queue_requires_authenticated_session() -> None: app = Flask(__name__) app.secret_key = "test-secret" @@ -107,10 +172,8 @@ def test_chat_prompt_queue_claim_reports_terminal_wrong_state(client) -> None: assert finish_resp.status_code == 200 claim_resp = client.post(f"/von/api/chat_prompt_queue/{queue_id}/claim") - assert claim_resp.status_code == 404 - payload = claim_resp.get_json() - assert payload["error_code"] == "wrong_state" - assert payload["details"]["current_status"] == "completed" + assert claim_resp.status_code == 410 + assert claim_resp.get_json()["error_code"] == "queue_claim_route_retired" def test_chat_prompt_queue_route_lists_recent_failed_items_separately(client) -> None: @@ -125,9 +188,6 @@ def test_chat_prompt_queue_route_lists_recent_failed_items_separately(client) -> assert create_resp.status_code == 201 queue_id = create_resp.get_json()["item"]["queue_id"] - claim_resp = client.post(f"/von/api/chat_prompt_queue/{queue_id}/claim") - assert claim_resp.status_code == 200 - finish_resp = client.post( f"/von/api/chat_prompt_queue/{queue_id}/finish", json={ @@ -155,8 +215,6 @@ def test_chat_prompt_queue_route_dismisses_failed_record_durably(client) -> None ) assert create_resp.status_code == 201 queue_id = create_resp.get_json()["item"]["queue_id"] - assert client.post(f"/von/api/chat_prompt_queue/{queue_id}/claim").status_code == 200 - failure_message = ( "Prompt queue record expired after being in progress for more than 24 hours." ) @@ -177,8 +235,49 @@ def test_chat_prompt_queue_route_dismisses_failed_record_durably(client) -> None list_resp = client.get("/von/api/chat_prompt_queue") assert list_resp.status_code == 200 - assert list_resp.get_json() == { - "success": True, - "items": [], - "recent_failed_items": [], - } + payload = list_resp.get_json() + assert payload["success"] is True + assert payload["items"] == [] + assert payload["recent_failed_items"] == [] + assert payload["turn_admission"]["active_global"] == 0 + assert payload["turn_admission"]["pending_admission"] == 0 + + +def test_queue_routes_cannot_release_a_live_server_bound_turn(client) -> None: + created = client.post( + "/von/api/chat_prompt_queue", + json={"prompt_raw": "Keep the durable fence", "session_id": "session-1"}, + ).get_json()["item"] + scope = chat_prompt_queue_service.build_queue_scope( + user_concept_id="#V#test_user", + organisation_concept_id="#V#test_org", + namespace="#V#test_user@test_org", + ) + chat_prompt_queue_service.bind_queue_record_to_turn( + scope=scope, + queue_id=created["queue_id"], + client_request_id="request-live", + attempt_id="attempt-live", + conversation_key=created["conversation_key"], + server_instance_id=SERVER_INSTANCE_ID, + ) + + requeue = client.post(f"/von/api/chat_prompt_queue/{created['queue_id']}/requeue") + cancel = client.delete(f"/von/api/chat_prompt_queue/{created['queue_id']}") + finish = client.post( + f"/von/api/chat_prompt_queue/{created['queue_id']}/finish", + json={"status": "completed"}, + ) + + assert requeue.status_code == 409 + assert requeue.get_json()["error_code"] == "conversation_turn_active" + assert cancel.status_code == 409 + assert cancel.get_json()["error_code"] == "conversation_turn_active" + assert finish.status_code == 404 + + chat_prompt_queue_service.finish_prompt_record( + scope=scope, + queue_id=created["queue_id"], + status=chat_prompt_queue_service.STATUS_COMPLETED, + attempt_id="attempt-live", + ) diff --git a/tests/backend/test_chat_prompt_queue_service.py b/tests/backend/test_chat_prompt_queue_service.py index e88083465..0875723d2 100644 --- a/tests/backend/test_chat_prompt_queue_service.py +++ b/tests/backend/test_chat_prompt_queue_service.py @@ -32,7 +32,10 @@ def test_queue_record_lifecycle_restores_active_items_only() -> None: ) assert queued["status"] == queue_service.STATUS_QUEUED - assert queue_service.list_active_queue_records(scope=scope)[0]["prompt_raw"] == "Second task" + assert ( + queue_service.list_active_queue_records(scope=scope)[0]["prompt_raw"] + == "Second task" + ) updated = queue_service.update_queued_record( scope=scope, @@ -45,10 +48,14 @@ def test_queue_record_lifecycle_restores_active_items_only() -> None: assert claimed["status"] == queue_service.STATUS_IN_PROGRESS assert claimed["attempt_count"] == 1 - requeued = queue_service.requeue_prompt_record(scope=scope, queue_id=queued["queue_id"]) + requeued = queue_service.requeue_prompt_record( + scope=scope, queue_id=queued["queue_id"] + ) assert requeued["status"] == queue_service.STATUS_QUEUED - reclaimed = queue_service.claim_queue_record(scope=scope, queue_id=queued["queue_id"]) + reclaimed = queue_service.claim_queue_record( + scope=scope, queue_id=queued["queue_id"] + ) assert reclaimed["attempt_count"] == 2 finished = queue_service.finish_prompt_record( @@ -60,6 +67,253 @@ def test_queue_record_lifecycle_restores_active_items_only() -> None: assert queue_service.list_active_queue_records(scope=scope) == [] +def test_turn_binding_correlates_attempt_and_serialises_one_conversation() -> None: + scope = queue_service.build_queue_scope( + user_concept_id="#V#user", + organisation_concept_id="#V#org", + namespace="#V#user@org", + ) + first = queue_service.create_queue_record( + scope=scope, + prompt_raw="First", + session_id="session-a", + client_request_id="request-a", + attempt_id="attempt-a", + ) + second = queue_service.create_queue_record( + scope=scope, + prompt_raw="Second", + session_id="session-b", + ) + + bound = queue_service.bind_queue_record_to_turn( + scope=scope, + queue_id=first["queue_id"], + client_request_id="request-a", + attempt_id="attempt-a", + conversation_key="conversation-key-a", + server_instance_id="server-a", + ) + parallel = queue_service.bind_queue_record_to_turn( + scope=scope, + queue_id=second["queue_id"], + client_request_id="request-b", + attempt_id="attempt-b", + conversation_key="conversation-key-b", + server_instance_id="server-a", + ) + + assert bound["status"] == queue_service.STATUS_IN_PROGRESS + assert bound["client_request_id"] == "request-a" + assert bound["attempt_id"] == "attempt-a" + assert bound["conversation_key"] == "conversation-key-a" + assert bound["server_instance_id"] == "server-a" + assert bound["lease_acquired_at"] is not None + assert parallel["conversation_key"] == "conversation-key-b" + + with pytest.raises(queue_service.ChatPromptQueueRecordNotFound): + queue_service.finish_prompt_record( + scope=scope, + queue_id=first["queue_id"], + status=queue_service.STATUS_COMPLETED, + ) + with pytest.raises(queue_service.ChatPromptQueueRecordNotFound): + queue_service.finish_prompt_record( + scope=scope, + queue_id=first["queue_id"], + status=queue_service.STATUS_COMPLETED, + attempt_id="wrong-attempt", + ) + + finished = queue_service.finish_prompt_record( + scope=scope, + queue_id=first["queue_id"], + status=queue_service.STATUS_COMPLETED, + attempt_id="attempt-a", + ) + repeated_finish = queue_service.finish_prompt_record( + scope=scope, + queue_id=first["queue_id"], + status=queue_service.STATUS_COMPLETED, + attempt_id="attempt-a", + ) + assert repeated_finish["completed_at"] == finished["completed_at"] + persisted = mongo_client.get_chat_prompt_queue_collection().find_one( + {"queue_id": first["queue_id"]} + ) + assert persisted is not None + assert "active_conversation_key" not in persisted + assert "lease_heartbeat_at" not in persisted + + +def test_shared_conversation_fifo_crosses_scopes_and_orders_timestamp_ties( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixed_now = datetime(2026, 8, 27, 12, 0, tzinfo=timezone.utc) + monkeypatch.setattr(queue_service, "_now", lambda: fixed_now) + owner_scope = queue_service.build_queue_scope( + user_concept_id="#V#owner", + organisation_concept_id="#V#org", + namespace="#V#owner@org", + ) + invitee_scope = queue_service.build_queue_scope( + user_concept_id="#V#invitee", + organisation_concept_id="#V#org", + namespace="#V#invitee@org", + ) + records = [ + queue_service.create_queue_record( + scope=owner_scope, + prompt_raw="Owner prompt", + session_id="shared-session", + conversation_key="canonical-shared-key", + ), + queue_service.create_queue_record( + scope=invitee_scope, + prompt_raw="Invitee prompt", + session_id="shared-session", + conversation_key="canonical-shared-key", + ), + ] + earlier, later = sorted(records, key=lambda item: item["queue_id"]) + scope_by_queue_id = { + records[0]["queue_id"]: owner_scope, + records[1]["queue_id"]: invitee_scope, + } + + with pytest.raises(queue_service.ConversationTurnAlreadyActive) as exc_info: + queue_service.bind_queue_record_to_turn( + scope=scope_by_queue_id[later["queue_id"]], + queue_id=later["queue_id"], + client_request_id="request-later", + conversation_key="canonical-shared-key", + server_instance_id="server-a", + ) + + assert exc_info.value.queue_id is None + bound = queue_service.bind_queue_record_to_turn( + scope=scope_by_queue_id[earlier["queue_id"]], + queue_id=earlier["queue_id"], + client_request_id="request-earlier", + conversation_key="canonical-shared-key", + server_instance_id="server-a", + ) + queue_service.finish_prompt_record( + scope=scope_by_queue_id[earlier["queue_id"]], + queue_id=earlier["queue_id"], + status=queue_service.STATUS_COMPLETED, + attempt_id=bound["attempt_id"], + ) + assert ( + queue_service.bind_queue_record_to_turn( + scope=scope_by_queue_id[later["queue_id"]], + queue_id=later["queue_id"], + client_request_id="request-later", + conversation_key="canonical-shared-key", + server_instance_id="server-a", + )["status"] + == queue_service.STATUS_IN_PROGRESS + ) + + +def test_queued_backlog_is_bounded_per_user_across_organisations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("VON_MAX_QUEUED_TURNS_GLOBAL", "10") + monkeypatch.setenv("VON_MAX_QUEUED_TURNS_PER_USER", "2") + scopes = [ + queue_service.build_queue_scope( + user_concept_id="#V#user", + organisation_concept_id=f"#V#org_{index}", + namespace=f"#V#user@org_{index}", + ) + for index in range(3) + ] + for index in range(2): + queue_service.create_queue_record( + scope=scopes[index], prompt_raw=f"Accepted {index}" + ) + + with pytest.raises(queue_service.ChatPromptQueueCapacityReached) as exc_info: + queue_service.create_queue_record(scope=scopes[2], prompt_raw="Rejected") + + assert exc_info.value.limit_kind == "user" + assert len(queue_service.list_active_queue_records(scope=scopes[0])) == 1 + assert len(queue_service.list_active_queue_records(scope=scopes[1])) == 1 + + +def test_queued_backlog_is_bounded_globally(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("VON_MAX_QUEUED_TURNS_GLOBAL", "2") + monkeypatch.setenv("VON_MAX_QUEUED_TURNS_PER_USER", "2") + for index in range(2): + scope = queue_service.build_queue_scope(user_concept_id=f"#V#user_{index}") + queue_service.create_queue_record(scope=scope, prompt_raw=f"Accepted {index}") + + with pytest.raises(queue_service.ChatPromptQueueCapacityReached) as exc_info: + queue_service.create_queue_record( + scope=queue_service.build_queue_scope(user_concept_id="#V#user_3"), + prompt_raw="Rejected", + ) + + assert exc_info.value.limit_kind == "global" + + +def test_requeue_releases_conversation_fence_and_request_correlation() -> None: + scope = queue_service.build_queue_scope( + user_concept_id="#V#user", + organisation_concept_id="#V#org", + namespace="#V#user@org", + ) + queued = queue_service.create_queue_record( + scope=scope, + prompt_raw="Retry me", + session_id="session-a", + ) + + queue_service.bind_queue_record_to_turn( + scope=scope, + queue_id=queued["queue_id"], + client_request_id="request-a", + attempt_id="attempt-a", + conversation_key="conversation-key-a", + server_instance_id="server-a", + ) + + with pytest.raises(queue_service.ConversationTurnAlreadyActive): + queue_service.requeue_prompt_record( + scope=scope, + queue_id=queued["queue_id"], + current_server_instance_id="server-a", + ) + with pytest.raises(queue_service.ConversationTurnAlreadyActive): + queue_service.cancel_prompt_record( + scope=scope, + queue_id=queued["queue_id"], + ) + + requeued = queue_service.requeue_prompt_record( + scope=scope, + queue_id=queued["queue_id"], + current_server_instance_id="server-b", + ) + assert requeued["status"] == queue_service.STATUS_QUEUED + assert requeued["client_request_id"] is None + assert requeued["attempt_id"] is None + + rebound = queue_service.bind_queue_record_to_turn( + scope=scope, + queue_id=queued["queue_id"], + client_request_id="request-b", + attempt_id="attempt-b", + conversation_key="conversation-key-a", + server_instance_id="server-b", + ) + assert rebound["status"] == queue_service.STATUS_IN_PROGRESS + assert rebound["client_request_id"] == "request-b" + assert rebound["attempt_id"] == "attempt-b" + assert rebound["server_instance_id"] == "server-b" + + def test_queue_scope_isolation() -> None: first_scope = queue_service.build_queue_scope( user_concept_id="#V#user_a", @@ -190,7 +444,9 @@ def test_legacy_scope_terminal_record_reports_wrong_state() -> None: ) with pytest.raises(queue_service.ChatPromptQueueRecordNotFound) as exc_info: - queue_service.claim_queue_record(scope=scope, queue_id="legacy-terminal-queue-1") + queue_service.claim_queue_record( + scope=scope, queue_id="legacy-terminal-queue-1" + ) assert exc_info.value.error_code == "wrong_state" assert exc_info.value.details["scope_match"] is True @@ -292,7 +548,9 @@ def test_list_recent_failed_queue_records_is_bounded_and_scoped() -> None: error="The final response did not return from Von.", ) - completed = queue_service.create_queue_record(scope=scope, prompt_raw="Completed task") + completed = queue_service.create_queue_record( + scope=scope, prompt_raw="Completed task" + ) queue_service.finish_prompt_record( scope=scope, queue_id=completed["queue_id"], @@ -313,7 +571,9 @@ def test_list_recent_failed_queue_records_is_bounded_and_scoped() -> None: coll = mongo_client.get_chat_prompt_queue_collection() assert coll is not None old_completed_at = datetime.now(timezone.utc) - timedelta(days=3) - stale_failed = queue_service.create_queue_record(scope=scope, prompt_raw="Old failed task") + stale_failed = queue_service.create_queue_record( + scope=scope, prompt_raw="Old failed task" + ) queue_service.finish_prompt_record( scope=scope, queue_id=stale_failed["queue_id"], @@ -332,7 +592,9 @@ def test_list_recent_failed_queue_records_is_bounded_and_scoped() -> None: assert recent[0]["last_error"] == "The final response did not return from Von." -def test_cancel_failed_record_is_scoped_durable_and_preserves_failure_evidence() -> None: +def test_cancel_failed_record_is_scoped_durable_and_preserves_failure_evidence() -> ( + None +): scope = queue_service.build_queue_scope( user_concept_id="#V#user", organisation_concept_id="#V#org", diff --git a/tests/backend/test_conversation_turn_admission_service.py b/tests/backend/test_conversation_turn_admission_service.py new file mode 100644 index 000000000..6f6e4992e --- /dev/null +++ b/tests/backend/test_conversation_turn_admission_service.py @@ -0,0 +1,489 @@ +from __future__ import annotations + +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from src.backend.db import mongo_client +from src.backend.services import chat_prompt_queue_service as queue_service +from src.backend.services.conversation_turn_admission_service import ( + ConversationTurnActive, + ConversationTurnAdmissionService, + ConversationTurnCapacityReached, + build_conversation_key, +) + + +@pytest.fixture(autouse=True) +def mock_db(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("VON_USE_MOCK_DB", "1") + monkeypatch.setenv("VON_DB_NAME", "test_conversation_turn_admission") + mongo_client.close_connection() + yield + mongo_client.close_connection() + + +def _scope(user: str = "#V#user", org: str = "#V#org") -> dict[str, str]: + return queue_service.build_queue_scope( + user_concept_id=user, + organisation_concept_id=org, + namespace=f"{user}@{org.removeprefix('#V#')}", + ) + + +def _key(session_id: str, owner: str = "#V#user") -> str: + return build_conversation_key( + owner_user_id=owner, + history_namespace=f"{owner}@org", + conversation_session_id=session_id, + ) + + +def _queued(scope: dict[str, str], session_id: str, prompt: str) -> str: + record = queue_service.create_queue_record( + scope=scope, + prompt_raw=prompt, + session_id=session_id, + ) + return str(record["queue_id"]) + + +def test_default_global_limit_preserves_http_thread_headroom( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("VON_MAX_ACTIVE_TURNS_GLOBAL", raising=False) + monkeypatch.setenv("VON_WAITRESS_THREADS", "16") + monkeypatch.setenv("VON_TURN_HTTP_HEADROOM_THREADS", "8") + + service = ConversationTurnAdmissionService() + + assert service.snapshot()["global_limit"] == 8 + assert service.snapshot()["configured_http_threads"] == 16 + assert service.snapshot()["available_http_thread_headroom"] == 8 + + +def test_explicit_global_limit_remains_observable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("VON_WAITRESS_THREADS", "32") + monkeypatch.setenv("VON_MAX_ACTIVE_TURNS_GLOBAL", "20") + + service = ConversationTurnAdmissionService() + + assert service.snapshot()["global_limit"] == 20 + assert service.snapshot()["available_http_thread_headroom"] == 12 + + +def test_distinct_conversations_run_concurrently_and_release_exactly() -> None: + scope = _scope() + service = ConversationTurnAdmissionService(per_user_limit=2, global_limit=4) + + first = service.acquire( + scope=scope, + prompt_raw="first", + session_id="conversation-a", + session_name=None, + client_request_id="request-a", + conversation_key=_key("conversation-a"), + queue_id=_queued(scope, "conversation-a", "first"), + ) + second = service.acquire( + scope=scope, + prompt_raw="second", + session_id="conversation-b", + session_name=None, + client_request_id="request-b", + conversation_key=_key("conversation-b"), + queue_id=_queued(scope, "conversation-b", "second"), + ) + + assert service.snapshot()["active_global"] == 2 + service.release(first, status=queue_service.STATUS_COMPLETED) + service.release(first, status=queue_service.STATUS_COMPLETED) + assert service.snapshot()["active_global"] == 1 + service.release(second, status=queue_service.STATUS_COMPLETED) + assert service.snapshot()["active_global"] == 0 + + +def test_release_retries_durable_terminalisation_before_freeing_capacity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + scope = _scope() + service = ConversationTurnAdmissionService(per_user_limit=1, global_limit=1) + token = service.acquire( + scope=scope, + prompt_raw="retry terminalisation", + session_id="conversation-a", + session_name=None, + client_request_id="request-a", + conversation_key=_key("conversation-a"), + queue_id=_queued(scope, "conversation-a", "retry terminalisation"), + ) + original_finish = queue_service.finish_prompt_record + attempts = 0 + + def _flaky_finish(**kwargs): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("temporary database failure") + return original_finish(**kwargs) + + monkeypatch.setattr(queue_service, "finish_prompt_record", _flaky_finish) + + with pytest.raises(RuntimeError, match="temporary database failure"): + service.release(token, status=queue_service.STATUS_COMPLETED) + assert service.snapshot()["active_global"] == 1 + assert token.released is False + + service.release(token, status=queue_service.STATUS_COMPLETED) + assert service.snapshot()["active_global"] == 0 + assert token.released is True + service.shutdown() + + +def test_release_reconciles_asynchronously_after_both_request_hooks_fail( + monkeypatch: pytest.MonkeyPatch, +) -> None: + scope = _scope() + service = ConversationTurnAdmissionService( + per_user_limit=1, + global_limit=1, + terminalisation_retry_initial_delay_sec=0.2, + terminalisation_retry_max_delay_sec=0.2, + ) + token = service.acquire( + scope=scope, + prompt_raw="reconcile terminalisation", + session_id="conversation-a", + session_name=None, + client_request_id="request-a", + conversation_key=_key("conversation-a"), + queue_id=_queued(scope, "conversation-a", "reconcile terminalisation"), + ) + original_finish = queue_service.finish_prompt_record + terminalised = threading.Event() + attempts: list[tuple[str, str | None, str | None]] = [] + + def _fail_hooks_then_reconcile(**kwargs): + attempts.append( + ( + kwargs["status"], + kwargs.get("error"), + kwargs.get("attempt_id"), + ) + ) + if len(attempts) <= 2: + raise RuntimeError("database unavailable through request teardown") + result = original_finish(**kwargs) + terminalised.set() + return result + + monkeypatch.setattr( + queue_service, + "finish_prompt_record", + _fail_hooks_then_reconcile, + ) + + try: + for _ in range(2): + with pytest.raises( + RuntimeError, + match="database unavailable through request teardown", + ): + service.release( + token, + status=queue_service.STATUS_CANCELLED, + error="cancelled at request boundary", + ) + + assert service.snapshot()["active_global"] == 1 + assert token.released is False + assert terminalised.wait(timeout=2) + assert service.snapshot()["active_global"] == 0 + assert token.released is True + assert len(attempts) == 3 + assert all( + status == queue_service.STATUS_CANCELLED + and error == "cancelled at request boundary" + and attempt_id == token.attempt_id + for status, error, attempt_id in attempts + ) + finally: + service.shutdown() + + +def test_same_conversation_has_exactly_one_racing_winner() -> None: + scope = _scope() + service = ConversationTurnAdmissionService(per_user_limit=4, global_limit=4) + queue_ids = [ + _queued(scope, "same-conversation", "first"), + _queued(scope, "same-conversation", "second"), + ] + barrier = threading.Barrier(2) + + def _acquire(index: int): + barrier.wait(timeout=2) + try: + return service.acquire( + scope=scope, + prompt_raw=f"prompt-{index}", + session_id="same-conversation", + session_name=None, + client_request_id=f"request-{index}", + conversation_key=_key("same-conversation"), + queue_id=queue_ids[index], + ) + except ConversationTurnActive: + return None + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(_acquire, range(2))) + + winners = [result for result in results if result is not None] + assert len(winners) == 1 + assert service.snapshot()["active_global"] == 1 + service.release(winners[0], status=queue_service.STATUS_COMPLETED) + + +def test_user_limit_spans_organisations_but_other_actor_can_progress() -> None: + service = ConversationTurnAdmissionService(per_user_limit=1, global_limit=3) + org_a = _scope(org="#V#org_a") + org_b = _scope(org="#V#org_b") + other = _scope(user="#V#other", org="#V#org_b") + + first = service.acquire( + scope=org_a, + prompt_raw="one", + session_id="a", + session_name=None, + client_request_id="request-a", + conversation_key=_key("a"), + queue_id=_queued(org_a, "a", "one"), + ) + with pytest.raises(ConversationTurnCapacityReached): + service.acquire( + scope=org_b, + prompt_raw="two", + session_id="b", + session_name=None, + client_request_id="request-b", + conversation_key=_key("b"), + queue_id=_queued(org_b, "b", "two"), + ) + + other_token = service.acquire( + scope=other, + prompt_raw="other", + session_id="other", + session_name=None, + client_request_id="request-other", + conversation_key=_key("other", owner="#V#other"), + queue_id=_queued(other, "other", "other"), + ) + assert service.snapshot()["active_global"] == 2 + service.release(first, status=queue_service.STATUS_COMPLETED) + service.release(other_token, status=queue_service.STATUS_COMPLETED) + + +def test_shared_owner_identity_builds_one_conversation_key() -> None: + owner_key = build_conversation_key( + owner_user_id="#V#owner", + history_namespace="#V#owner@org", + conversation_session_id="shared-session", + ) + invitee_view_key = build_conversation_key( + owner_user_id="#V#owner", + history_namespace="#V#owner@org", + conversation_session_id="shared-session", + ) + + assert owner_key == invitee_view_key + + +def test_ephemeral_turns_share_global_capacity_and_conversation_single_flight() -> None: + service = ConversationTurnAdmissionService(per_user_limit=2, global_limit=2) + first = service.acquire_ephemeral( + actor_capacity_key="anonymous-browser-a", + conversation_key="anonymous-conversation-a", + client_request_id="anonymous-request-a", + ) + second = service.acquire_ephemeral( + actor_capacity_key="anonymous-browser-b", + conversation_key="anonymous-conversation-b", + client_request_id="anonymous-request-b", + ) + + assert service.snapshot()["active_global"] == 2 + with pytest.raises(ConversationTurnCapacityReached): + service.acquire_ephemeral( + actor_capacity_key="anonymous-browser-c", + conversation_key="anonymous-conversation-c", + client_request_id="anonymous-request-c", + ) + # Free global capacity so the next assertion discriminates the + # conversation fence from the global ceiling. + service.release(second, status=queue_service.STATUS_COMPLETED) + with pytest.raises(ConversationTurnActive): + service.acquire_ephemeral( + actor_capacity_key="anonymous-browser-a", + conversation_key="anonymous-conversation-a", + client_request_id="anonymous-request-a-2", + ) + + service.release(first, status=queue_service.STATUS_COMPLETED) + assert service.snapshot()["active_global"] == 0 + + +@pytest.mark.parametrize("user_count", [10, 20, 50]) +def test_representative_user_load_admits_and_recovers_in_bounded_waves( + user_count: int, +) -> None: + """Representative bursts use the production 24-global/4-user bounds.""" + + # Prewarm the lazy mock client and indexes before worker threads. Without + # this, concurrent first access can create separate Mongo-mock clients and + # test the fixture bootstrap rather than turn admission. + assert mongo_client.get_chat_prompt_queue_collection() is not None + service = ConversationTurnAdmissionService(per_user_limit=4, global_limit=24) + work = [] + for index in range(user_count): + user = f"#V#load_user_{index}" + scope = _scope(user=user) + session_id = f"load-conversation-{index}" + work.append( + ( + index, + user, + scope, + session_id, + _queued(scope, session_id, f"prompt {index}"), + ) + ) + + def _admit_wave(items): + barrier = threading.Barrier(len(items)) + + def _acquire(item): + index, user, scope, session_id, queue_id = item + barrier.wait(timeout=5) + try: + return item, service.acquire( + scope=scope, + prompt_raw=f"prompt {index}", + session_id=session_id, + session_name=None, + client_request_id=f"load-request-{index}", + conversation_key=_key(session_id, owner=user), + queue_id=queue_id, + ) + except ConversationTurnCapacityReached: + return item, None + + with ThreadPoolExecutor(max_workers=len(items)) as executor: + return list(executor.map(_acquire, items)) + + first_wave = _admit_wave(work) + admitted = [token for _item, token in first_wave if token is not None] + rejected = [item for item, token in first_wave if token is None] + assert len(admitted) == min(user_count, 24) + assert len(rejected) == max(0, user_count - 24) + assert service.snapshot()["active_global"] == len(admitted) + + for token in admitted: + service.release(token, status=queue_service.STATUS_COMPLETED) + assert service.snapshot()["active_global"] == 0 + + while rejected: + next_items, rejected = rejected[:24], rejected[24:] + recovery_wave = _admit_wave(next_items) + recovery_tokens = [token for _item, token in recovery_wave if token is not None] + assert len(recovery_tokens) == len(next_items) + for token in recovery_tokens: + service.release(token, status=queue_service.STATUS_COMPLETED) + + assert service.snapshot()["active_global"] == 0 + assert service.snapshot()["active_user_count"] == 0 + + +def test_twenty_users_each_keep_one_active_and_one_queued_turn_visible() -> None: + """Twenty users retain two isolated active-or-queued turns each.""" + + assert mongo_client.get_chat_prompt_queue_collection() is not None + user_count = 20 + service = ConversationTurnAdmissionService(per_user_limit=4, global_limit=24) + work = [] + for index in range(user_count): + user = f"#V#two_turn_user_{index}" + scope = _scope(user=user) + active_session = f"active-conversation-{index}" + active_queue_id = _queued(scope, active_session, f"active prompt {index}") + queued_queue_id = _queued( + scope, + f"queued-conversation-{index}", + f"queued prompt {index}", + ) + work.append( + (index, user, scope, active_session, active_queue_id, queued_queue_id) + ) + + barrier = threading.Barrier(user_count) + + def _acquire(item): + index, user, scope, session_id, queue_id, _queued_queue_id = item + barrier.wait(timeout=5) + return service.acquire( + scope=scope, + prompt_raw=f"active prompt {index}", + session_id=session_id, + session_name=None, + client_request_id=f"two-turn-request-{index}", + conversation_key=_key(session_id, owner=user), + queue_id=queue_id, + ) + + with ThreadPoolExecutor(max_workers=user_count) as executor: + tokens = list(executor.map(_acquire, work)) + + assert service.snapshot()["active_global"] == user_count + for item in work: + _index, _user, scope, _session_id, active_queue_id, queued_queue_id = item + visible = queue_service.list_active_queue_records(scope=scope) + assert {record["queue_id"] for record in visible} == { + active_queue_id, + queued_queue_id, + } + assert {record["status"] for record in visible} == { + queue_service.STATUS_IN_PROGRESS, + queue_service.STATUS_QUEUED, + } + + first_scope_ids = { + record["queue_id"] + for record in queue_service.list_active_queue_records(scope=work[0][2]) + } + assert work[1][4] not in first_scope_ids + assert work[1][5] not in first_scope_ids + + for token in tokens: + service.release(token, status=queue_service.STATUS_COMPLETED) + assert service.snapshot()["active_global"] == 0 + + for item in work: + _index, _user, scope, _session_id, active_queue_id, queued_queue_id = item + visible = queue_service.list_active_queue_records(scope=scope) + assert [record["queue_id"] for record in visible] == [queued_queue_id] + queue_service.finish_prompt_record( + scope=scope, + queue_id=queued_queue_id, + status=queue_service.STATUS_CANCELLED, + ) + assert queue_service.list_active_queue_records(scope=scope) == [] + persisted_active = mongo_client.get_chat_prompt_queue_collection().find_one( + {"queue_id": active_queue_id} + ) + assert persisted_active is not None + assert persisted_active["status"] == queue_service.STATUS_COMPLETED + + assert service.snapshot()["active_user_count"] == 0 diff --git a/tests/backend/test_tool_progress_liveness.py b/tests/backend/test_tool_progress_liveness.py index 2606e7ae9..94eda23d8 100644 --- a/tests/backend/test_tool_progress_liveness.py +++ b/tests/backend/test_tool_progress_liveness.py @@ -16,6 +16,10 @@ queued_tool_progress_state_count, reset_tool_progress_persistence_queue_for_tests, ) +from src.backend.services.window_session_context_service import ( + WindowSessionContext, + get_window_session_store, +) _ORIGINAL_USE_MOCK_DB = os.environ.get("VON_USE_MOCK_DB") _ORIGINAL_DB_NAME = os.environ.get("VON_DB_NAME") @@ -41,6 +45,33 @@ def _progress_app() -> Flask: return app +def _bind_authenticated_progress_scope( + app: Flask, + *, + user_id: str = "#V#test_user", + organisation_id: str = "#V#test_org", + namespace: str = "#V#test_user@test_org", + window_session_id: str = "ws_exact_progress", +): + get_window_session_store().set( + WindowSessionContext( + window_session_id=window_session_id, + user_id=user_id, + organisation_concept_id=organisation_id, + namespace=namespace, + ) + ) + client = app.test_client() + with client.session_transaction() as sess: + sess["user_concept_id"] = user_id + scope_key = von_routes._build_authenticated_tool_progress_scope_key( + user_concept_id=user_id, + organisation_concept_id=organisation_id, + namespace=namespace, + ) + return client, scope_key, window_session_id + + def _relation_semantic_operation( *, lifecycle_status: str, @@ -291,16 +322,12 @@ def test_progress_endpoint_reads_persisted_state_after_local_cache_miss( monkeypatch, ) -> None: app = _progress_app() - monkeypatch.setattr( - "src.backend.security.access_control.get_effective_user_concept_id", - lambda: "#V#test_user", - ) monkeypatch.setattr( "src.backend.server.routes.von_routes.get_show_tool_use_during_thinking", lambda: True, ) - scope_key = "user:#V#test_user" + client, scope_key, window_session_id = _bind_authenticated_progress_scope(app) semantic_operation = _relation_semantic_operation(lifecycle_status="running") von_routes._set_tool_progress( scope_key, @@ -318,7 +345,10 @@ def test_progress_endpoint_reads_persisted_state_after_local_cache_miss( with von_routes._TOOL_PROGRESS_LOCK: von_routes._TOOL_PROGRESS.clear() - response = app.test_client().get("/von/progress/req-persisted") + response = client.get( + "/von/progress/req-persisted", + headers={"X-Von-Window-Session": window_session_id}, + ) assert response.status_code == 200 body = response.get_json() assert isinstance(body, dict) @@ -557,12 +587,10 @@ def test_progress_endpoint_returns_explanatory_pending_payload(monkeypatch) -> N ) response = app.test_client().get("/von/progress/req-pending") - assert response.status_code == 202 + assert response.status_code == 401 body = response.get_json() assert isinstance(body, dict) - assert body.get("status") == "pending" - assert body.get("pending_reason") == "no_visible_progress_state" - assert "No live progress state is visible yet" in str(body.get("result_summary")) + assert body.get("error") == "progress_authentication_required" def test_progress_endpoint_uses_window_scope_for_anonymous_requests( @@ -604,11 +632,8 @@ def test_progress_endpoint_uses_window_scope_for_anonymous_requests( "/von/progress/req-window-scope", headers={"X-Von-Window-Session": window_session_id}, ) - assert response.status_code == 200 - body = response.get_json() - assert isinstance(body, dict) - assert body.get("stage") == "evidence_read" - assert body.get("phase_label") == "Reading evidence" + assert response.status_code == 401 + assert response.get_json()["error"] == "progress_authentication_required" def test_progress_endpoint_uses_header_user_scope_without_session_lookup( @@ -641,11 +666,8 @@ def test_progress_endpoint_uses_header_user_scope_without_session_lookup( f"/von/progress/{request_id}", headers={"X-User-Concept-ID": "#V#test_user"}, ) - assert response.status_code == 200 - body = response.get_json() - assert isinstance(body, dict) - assert body.get("stage") == "model_call" - assert body.get("phase_label") == "Generating response" + assert response.status_code == 401 + assert response.get_json()["error"] == "progress_authentication_required" def test_progress_endpoint_uses_window_header_without_session_lookup( @@ -682,10 +704,8 @@ def test_progress_endpoint_uses_window_header_without_session_lookup( "X-User-Concept-ID": "#V#test_user", }, ) - assert response.status_code == 200 - body = response.get_json() - assert isinstance(body, dict) - assert body.get("stage") == "evidence_read" + assert response.status_code == 401 + assert response.get_json()["error"] == "progress_authentication_required" def test_progress_endpoint_recovers_window_scope_after_auth_scope_shift( @@ -723,12 +743,57 @@ def test_progress_endpoint_recovers_window_scope_after_auth_scope_shift( "X-User-Concept-ID": "#V#test_user", }, ) - assert response.status_code == 200 - body = response.get_json() - assert isinstance(body, dict) - assert body.get("stage") == "model_call" - assert body.get("resolved_scope_key") == f"anon:window:{window_session_id}" - assert body.get("progress_source") == "alternate_scope_fallback" + assert response.status_code == 401 + assert response.get_json()["error"] == "progress_authentication_required" + + +def test_progress_endpoint_rejects_unknown_and_wrong_exact_window_scope( + monkeypatch, +) -> None: + app = _progress_app() + monkeypatch.setattr( + "src.backend.server.routes.von_routes.get_show_tool_use_during_thinking", + lambda: True, + ) + client, scope_key, window_session_id = _bind_authenticated_progress_scope(app) + request_id = "req-exact-scope" + von_routes._set_tool_progress( + scope_key, + request_id, + { + "status": "thinking", + "phase": "model_call", + "request_id": request_id, + }, + ) + get_window_session_store().set( + WindowSessionContext( + window_session_id="ws-other-org", + user_id="#V#test_user", + organisation_concept_id="#V#other_org", + namespace="#V#test_user@other_org", + ) + ) + + visible = client.get( + f"/von/progress/{request_id}", + headers={"X-Von-Window-Session": window_session_id}, + ) + wrong_scope = client.get( + f"/von/progress/{request_id}", + headers={"X-Von-Window-Session": "ws-other-org"}, + ) + unknown_window = client.get( + f"/von/progress/{request_id}", + headers={"X-Von-Window-Session": "ws-unknown"}, + ) + + assert visible.status_code == 200 + assert visible.get_json()["stage"] == "model_call" + assert wrong_scope.status_code == 404 + assert wrong_scope.get_json() == {"error": "progress_scope_mismatch"} + assert unknown_window.status_code == 409 + assert unknown_window.get_json() == {"error": "progress_window_scope_unavailable"} def test_scope_alias_registration_copies_current_state(monkeypatch) -> None: diff --git a/tests/backend/test_von_background_durable_reconciliation.py b/tests/backend/test_von_background_durable_reconciliation.py index 8d178d69c..a103ea201 100644 --- a/tests/backend/test_von_background_durable_reconciliation.py +++ b/tests/backend/test_von_background_durable_reconciliation.py @@ -7,6 +7,7 @@ from flask import Flask from src.backend.services.background_task_service import TaskStatus +from src.backend.services.chat_prompt_queue_service import build_queue_scope from src.backend.workflows.durable.models import WorkflowInstanceStatus @@ -31,6 +32,8 @@ def mark_terminal_external(self, task_id: str, **kwargs: Any) -> TaskStatus: progress=dict(kwargs.get("progress") or {}), session_id=kwargs.get("session_id"), user_id=kwargs.get("user_id"), + organisation_id=kwargs.get("organisation_id"), + namespace=kwargs.get("namespace"), ) return self.status @@ -64,6 +67,22 @@ def _make_app( "src.backend.server.routes.von_routes.get_instance_manager", lambda: manager, ) + scope = build_queue_scope(user_concept_id="#V#task_test_user") + if registry.status is not None: + registry.status.user_id = scope["user_concept_id"] + registry.status.organisation_id = scope["organisation_concept_id"] + registry.status.namespace = scope["namespace"] + for instance in { + id(manager.instance): manager.instance, + id(manager.hydrated_instance): manager.hydrated_instance, + }.values(): + instance.user_id = scope["user_concept_id"] + instance.org_id = scope["organisation_concept_id"] + instance.namespace = scope["namespace"] + monkeypatch.setattr( + "src.backend.security.access_control.get_effective_user_concept_id_with_source", + lambda: ("#V#task_test_user", "authenticated_session"), + ) app = Flask(__name__) app.secret_key = "test-secret" @@ -113,6 +132,70 @@ def test_background_status_reconciles_terminal_durable_turn(monkeypatch) -> None assert manager.calls[0]["source_event_id"] == "turn-123" +def test_background_task_id_is_not_visible_to_another_actor(monkeypatch) -> None: + status = TaskStatus( + task_id="private-turn", + status="running", + created_at=datetime.now(timezone.utc), + user_id="#V#task_test_user", + namespace="#V#task_test_user", + ) + instance = SimpleNamespace( + instance_id="private-instance", + status=WorkflowInstanceStatus.COMPLETED, + current_state="responded", + error=None, + source_event_id="private-turn", + inputs={"conversation_session_id": "private-session"}, + outputs={"response": "private"}, + ) + app = _make_app( + monkeypatch, + _RouteRegistry(status), + _TerminalTurnManager(instance), + ) + monkeypatch.setattr( + "src.backend.security.access_control.get_effective_user_concept_id_with_source", + lambda: ("#V#other_user", "authenticated_session"), + ) + + response = app.test_client().get("/von/api/task/status/private-turn") + + assert response.status_code == 404 + + +def test_background_task_id_is_not_visible_in_another_organisation( + monkeypatch, +) -> None: + status = TaskStatus( + task_id="org-private-turn", + status="running", + created_at=datetime.now(timezone.utc), + ) + instance = SimpleNamespace( + instance_id="org-private-instance", + status=WorkflowInstanceStatus.COMPLETED, + current_state="responded", + error=None, + source_event_id="org-private-turn", + inputs={"conversation_session_id": "private-session"}, + outputs={"response": "private"}, + ) + app = _make_app( + monkeypatch, + _RouteRegistry(status), + _TerminalTurnManager(instance), + ) + client = app.test_client() + with client.session_transaction() as flask_session: + flask_session["organisation_concept_id"] = "#V#other_org" + flask_session["namespace"] = "#V#task_test_user@other_org" + + response = client.get("/von/api/task/result/org-private-turn") + + assert response.status_code == 404 + + def test_background_status_does_not_call_lifecycle_completion_user_success( monkeypatch, ) -> None: @@ -367,8 +450,7 @@ def test_background_result_prefers_user_answer_over_machine_json_response( "request_id": "turn-json", "session_id": "session-json", "response": ( - '{"confidence": 1.0, "workflow_id": ' - '"#V#turn_completion_gate_workflow"}' + '{"confidence": 1.0, "workflow_id": "#V#turn_completion_gate_workflow"}' ), "selected_workflow_user_response": "Grounded Jira answer.", "final_response": "Grounded Jira answer.", @@ -418,12 +500,10 @@ def test_background_result_uses_completion_report_before_machine_json( "request_id": "turn-report", "session_id": "session-report", "response": ( - '{"confidence": 1.0, "workflow_id": ' - '"#V#turn_completion_gate_workflow"}' + '{"confidence": 1.0, "workflow_id": "#V#turn_completion_gate_workflow"}' ), "final_response": ( - '{"confidence": 1.0, "workflow_id": ' - '"#V#turn_completion_gate_workflow"}' + '{"confidence": 1.0, "workflow_id": "#V#turn_completion_gate_workflow"}' ), "completion_report": { "response_text": "Here is the grounded completion report answer." @@ -505,6 +585,8 @@ def test_background_generate_success_body_marks_task_completed_before_persistenc request_id="turn-ready", session_id="session-ready", user_id="#V#michael_witbrock", + organisation_id="#V#von_org", + namespace="#V#michael_witbrock@von_org", response_text="Ready answer", ) @@ -512,6 +594,8 @@ def test_background_generate_success_body_marks_task_completed_before_persistenc assert registry.status.status == "completed" assert registry.status.session_id == "session-ready" assert registry.status.user_id == "#V#michael_witbrock" + assert registry.status.organisation_id == "#V#von_org" + assert registry.status.namespace == "#V#michael_witbrock@von_org" assert registry.status.result == { "response": "Ready answer", "request_id": "turn-ready", diff --git a/tests/backend/test_von_generate_background_submission.py b/tests/backend/test_von_generate_background_submission.py index 5bb22a7e8..285402f30 100644 --- a/tests/backend/test_von_generate_background_submission.py +++ b/tests/backend/test_von_generate_background_submission.py @@ -7,6 +7,7 @@ from typing import Any, Mapping, cast from src.backend.services.adaptive_turn_service import AdaptiveTurnResult +from src.backend.services.background_task_service import BackgroundTaskCapacityReached from src.backend.services.tool_progress_store_service import ( reset_tool_progress_persistence_queue_for_tests, ) @@ -27,7 +28,9 @@ def _clear_live_progress_state(): class _DummyLLM: def generate(self, *_args, **_kwargs): - raise AssertionError("LLM generate() should not be called directly in this test") + raise AssertionError( + "LLM generate() should not be called directly in this test" + ) class _StubAdaptiveTurn: @@ -60,6 +63,11 @@ def submit_task( progress_callback, session_id, user_id, + organisation_id, + namespace, + queue_id, + client_request_id, + attempt_id, ): self.calls.append( { @@ -68,6 +76,11 @@ def submit_task( "progress_callback": progress_callback, "session_id": session_id, "user_id": user_id, + "organisation_id": organisation_id, + "namespace": namespace, + "queue_id": queue_id, + "client_request_id": client_request_id, + "attempt_id": attempt_id, } ) return _SubmittedTaskStatus() @@ -128,7 +141,7 @@ def _make_app(monkeypatch, adaptive_turn, task_registry) -> Flask: ) monkeypatch.setattr( "src.backend.security.access_control.get_effective_user_concept_id", - lambda: None, + lambda: "#V#test_user", ) monkeypatch.setattr( "src.backend.services.workflow_discovery_service.discover_workflows_for_turn", @@ -145,6 +158,11 @@ def _make_app(monkeypatch, adaptive_turn, task_registry) -> Flask: "_resolve_shared_conversation_owner", lambda **_kwargs: (None, None), ) + monkeypatch.setattr( + von_routes, + "_load_conversation_session_state_fail_soft", + lambda **_kwargs: ([], None, None, 0, [], {}), + ) app = Flask(__name__) app.secret_key = "test-secret" @@ -153,6 +171,19 @@ def _make_app(monkeypatch, adaptive_turn, task_registry) -> Flask: app.config["CONTEXT"] = [] app.config["INTERNAL_MCP_ORCHESTRATOR"] = object() app.config["INTERNAL_MCP_GATEWAY"] = None + from src.backend.services.window_session_context_service import ( + get_window_session_store, + set_window_organisation, + ) + + get_window_session_store().delete("window-123") + set_window_organisation( + "window-123", + "#V#test_org", + "member", + "#V#test_user@test_org", + user_id="#V#test_user", + ) return app @@ -222,6 +253,64 @@ def fail_progress_setting_lookup() -> bool: assert body["task_id"] == submitted["task_id"] +def test_background_generate_returns_typed_capacity_backpressure(monkeypatch): + task_registry = _CapturingTaskRegistry() + + def _capacity_reached(**_kwargs): + raise BackgroundTaskCapacityReached("user background capacity reached") + + task_registry.submit_task = _capacity_reached + app = _make_app( + monkeypatch, + _StubAdaptiveTurn( + AdaptiveTurnResult( + response_text="must not run", + extra_messages=(), + tool_invocations=(), + aux_llm_calls=(), + ) + ), + task_registry, + ) + + response = app.test_client().post( + "/von/generate", + json={ + "prompt": "Run this in the background", + "background": True, + "model": "gpt-5.4-nano", + }, + headers={"X-Von-Window-Session": "window-123"}, + ) + + assert response.status_code == 429 + assert response.headers["Retry-After"] == "1" + assert response.get_json()["error"] == "background_task_capacity_reached" + assert response.get_json()["retryable"] is True + + +def test_generate_requires_rebind_for_unknown_window_context(monkeypatch): + adaptive_turn = _StubAdaptiveTurn( + AdaptiveTurnResult( + response_text="must not run", + extra_messages=(), + tool_invocations=(), + aux_llm_calls=(), + ) + ) + app = _make_app(monkeypatch, adaptive_turn, _CapturingTaskRegistry()) + + response = app.test_client().post( + "/von/generate", + json={"prompt": "Use this tab's organisation"}, + headers={"X-Von-Window-Session": "missing-after-restart"}, + ) + + assert response.status_code == 409 + assert response.get_json()["error"] == "window_context_unavailable" + assert adaptive_turn.calls == [] + + def test_background_generate_reentry_passes_authorised_inputs_to_adaptive_turn( monkeypatch, ): @@ -271,6 +360,54 @@ def test_background_generate_reentry_passes_authorised_inputs_to_adaptive_turn( assert isinstance(llm_debug.get("turn_execution_record"), dict) +def test_background_generate_freezes_org_scope_when_tab_switches(monkeypatch): + from src.backend.services.window_session_context_service import ( + set_window_organisation, + ) + + adaptive_turn = _StubAdaptiveTurn( + AdaptiveTurnResult( + response_text="ok", + extra_messages=(), + tool_invocations=(), + aux_llm_calls=(), + ) + ) + task_registry = _CapturingTaskRegistry() + app = _make_app(monkeypatch, adaptive_turn, task_registry) + + response = app.test_client().post( + "/von/generate", + json={ + "prompt": "Keep this task in its submitting organisation", + "background": True, + "model": "gpt-5.4-nano", + }, + headers={"X-Von-Window-Session": "window-123"}, + ) + assert response.status_code == 202 + + submitted = cast(dict[str, Any], task_registry.calls[0]) + assert submitted["organisation_id"] == "#V#test_org" + assert submitted["namespace"] == "#V#test_user@test_org" + + set_window_organisation( + "window-123", + "#V#other_org", + "member", + "#V#test_user@other_org", + user_id="#V#test_user", + ) + + task_callable = submitted["callable"] + assert callable(task_callable) + task_callable() + + assert adaptive_turn.calls + assert adaptive_turn.calls[0]["org_concept_id"] == "#V#test_org" + assert adaptive_turn.calls[0]["user_namespace"] == "#V#test_user@test_org" + + def test_background_generate_preserves_adaptive_non_success(monkeypatch): adaptive_result = AdaptiveTurnResult( response_text="The model call failed.", @@ -317,6 +454,7 @@ def fake_generate(): payload = request.get_json() assert isinstance(payload, dict) captured_payload.update(payload) + captured_payload["_frozen_role"] = session.get("role_in_org") return jsonify({"response": "ok"}), 200 monkeypatch.setattr(von_routes, "generate", fake_generate) @@ -340,6 +478,13 @@ def fake_generate(): request_headers=cast(Mapping[str, Any], request.headers), session_snapshot=dict(session), request_id="task-123", + actor_scope={ + "user_concept_id": "#V#test_user", + "organisation_concept_id": "#V#test_org", + "namespace": "#V#test_user@test_org", + "role_in_org": "member", + }, + conversation_session_id="session-123", ) assert status_code == 202 @@ -356,9 +501,10 @@ def fake_generate(): assert captured_payload["background_task_id"] == "task-123" assert captured_payload["background_progress"] is True assert captured_payload["workflow_inputs"] == {"gmail_max_results": 1} + assert captured_payload["_frozen_role"] == "member" -def test_background_generate_reentry_projects_progress_for_mcp_live_readback( +def test_background_generate_reentry_does_not_publish_before_exact_scope_resolution( monkeypatch, ): from src.backend.services.turn_execution_live_progress_service import ( @@ -406,13 +552,7 @@ def test_background_generate_reentry_projects_progress_for_mcp_live_readback( request_id="task-123", window_session_id="window-123", ) - assert live_progress is not None - assert live_progress["success"] is True - assert live_progress["request_id"] == "task-123" - assert live_progress["resolved_scope_key"] == "anon:window:window-123" - assert live_progress["progress_source"] == "tool_progress_state" - assert live_progress["status"] == "thinking" - assert live_progress["stage"] == "context_build" + assert live_progress is None def test_generate_passes_authorised_inputs_to_adaptive_turn(monkeypatch): @@ -505,9 +645,7 @@ def test_orchestrator_projects_namespaced_workflow_launch_inputs_safely() -> Non assert payload["prompt"] == "original prompt" assert "workflow_id" not in payload assert "__private" not in payload - assert payload["namespaced_workflow_launch_inputs_applied"] == [ - "gmail_max_results" - ] + assert payload["namespaced_workflow_launch_inputs_applied"] == ["gmail_max_results"] def test_normalise_background_generate_result_preserves_json_payload() -> None: @@ -559,11 +697,13 @@ def test_resolve_generate_requested_model_prefers_explicit_request_model(monkeyp ) monkeypatch.setattr(von_routes, "resolve_llm_setting", lambda **_kwargs: None) - model_name, client_type, model_parameters = von_routes._resolve_generate_requested_model( - {"model": "gpt-5.4-nano"}, - user_concept_id=None, - org_concept_id=None, - configured_model=None, + model_name, client_type, model_parameters = ( + von_routes._resolve_generate_requested_model( + {"model": "gpt-5.4-nano"}, + user_concept_id=None, + org_concept_id=None, + configured_model=None, + ) ) assert model_name == "gpt-5.4-nano" @@ -583,11 +723,13 @@ def test_resolve_generate_requested_model_ignores_browser_object_request_model( ) monkeypatch.setattr(von_routes, "resolve_llm_setting", lambda **_kwargs: None) - model_name, client_type, model_parameters = von_routes._resolve_generate_requested_model( - {"model": "openai:[object PointerEvent]"}, - user_concept_id="#V#test_user", - org_concept_id=None, - configured_model=None, + model_name, client_type, model_parameters = ( + von_routes._resolve_generate_requested_model( + {"model": "openai:[object PointerEvent]"}, + user_concept_id="#V#test_user", + org_concept_id=None, + configured_model=None, + ) ) assert model_name == "gpt-5.4-mini" @@ -607,11 +749,13 @@ def test_resolve_generate_requested_model_overrides_scoped_setting_with_explicit ) monkeypatch.setattr(von_routes, "resolve_llm_setting", lambda **_kwargs: None) - model_name, client_type, model_parameters = von_routes._resolve_generate_requested_model( - {"model": "ollama:llama3.1:8b"}, - user_concept_id="#V#test_user", - org_concept_id=None, - configured_model=None, + model_name, client_type, model_parameters = ( + von_routes._resolve_generate_requested_model( + {"model": "ollama:llama3.1:8b"}, + user_concept_id="#V#test_user", + org_concept_id=None, + configured_model=None, + ) ) assert model_name == "llama3.1:8b" @@ -631,11 +775,13 @@ def test_resolve_generate_requested_model_honours_explicit_provider_field( ) monkeypatch.setattr(von_routes, "resolve_llm_setting", lambda **_kwargs: None) - model_name, client_type, model_parameters = von_routes._resolve_generate_requested_model( - {"model": "gemma4:e4b", "model_provider": "ollama"}, - user_concept_id="#V#test_user", - org_concept_id=None, - configured_model=None, + model_name, client_type, model_parameters = ( + von_routes._resolve_generate_requested_model( + {"model": "gemma4:e4b", "model_provider": "ollama"}, + user_concept_id="#V#test_user", + org_concept_id=None, + configured_model=None, + ) ) assert model_name == "gemma4:e4b" @@ -667,11 +813,13 @@ def test_resolve_generate_requested_model_preserves_scoped_model_parameters( }, ) - model_name, client_type, model_parameters = von_routes._resolve_generate_requested_model( - {}, - user_concept_id="#V#test_user", - org_concept_id=None, - configured_model=None, + model_name, client_type, model_parameters = ( + von_routes._resolve_generate_requested_model( + {}, + user_concept_id="#V#test_user", + org_concept_id=None, + configured_model=None, + ) ) assert model_name == "gpt-5.5" diff --git a/tests/backend/test_von_route_json_safe_payloads.py b/tests/backend/test_von_route_json_safe_payloads.py index ec76793da..0b256a102 100644 --- a/tests/backend/test_von_route_json_safe_payloads.py +++ b/tests/backend/test_von_route_json_safe_payloads.py @@ -6,9 +6,10 @@ from pathlib import Path from types import SimpleNamespace -from flask import Flask +from flask import Flask, g, jsonify import src.backend.server.routes.von_routes as von_routes +from src.backend.services.request_progress_service import CancellationRequested @dataclass @@ -39,6 +40,75 @@ def test_json_safe_response_payload_converts_python_native_debug_values() -> Non json.dumps(safe) +def test_turn_admission_release_retries_the_original_terminal_disposition( + monkeypatch, +) -> None: + app = Flask(__name__) + attempted_statuses: list[str] = [] + token = SimpleNamespace( + released=False, + queue_id="queue-1", + client_request_id="request-1", + pending_terminal_status=None, + pending_terminal_error=None, + ) + + def _flaky_release(_token, *, status, error=None): + attempted_statuses.append(status) + if len(attempted_statuses) == 1: + raise RuntimeError("unknown write acknowledgement") + _token.released = True + + monkeypatch.setattr( + von_routes.conversation_turn_admission_service, + "release", + _flaky_release, + ) + + with app.test_request_context("/von/generate"): + setattr(g, von_routes._TURN_ADMISSION_CONTEXT_KEY, token) + response = jsonify( + {"success": False, "terminal_status": "cancelled", "error": "stopped"} + ) + von_routes._release_request_turn_admission(response=response) + von_routes._release_request_turn_admission( + exception=RuntimeError("conflicting teardown observation") + ) + + assert attempted_statuses == ["cancelled", "cancelled"] + assert token.pending_terminal_error == "stopped" + + +def test_turn_admission_maps_cancellation_exception_to_cancelled(monkeypatch) -> None: + app = Flask(__name__) + disposition: dict[str, object] = {} + token = SimpleNamespace( + released=False, + queue_id="queue-1", + client_request_id="request-1", + pending_terminal_status=None, + pending_terminal_error=None, + ) + + def _capture_release(_token, *, status, error=None): + disposition.update(status=status, error=error) + _token.released = True + + monkeypatch.setattr( + von_routes.conversation_turn_admission_service, + "release", + _capture_release, + ) + + with app.test_request_context("/von/generate"): + setattr(g, von_routes._TURN_ADMISSION_CONTEXT_KEY, token) + von_routes._release_request_turn_admission( + exception=CancellationRequested(task_id="request-1") + ) + + assert disposition["status"] == "cancelled" + + def test_task_status_endpoint_jsonifies_set_payloads(monkeypatch) -> None: app = Flask(__name__) app.register_blueprint(von_routes.von_bp, url_prefix="/von") @@ -54,8 +124,17 @@ def test_task_status_endpoint_jsonifies_set_payloads(monkeypatch) -> None: monkeypatch.setattr( von_routes.background_task_registry, - "get_task_status", - lambda task_id: status, + "get_task_status_for_scope", + lambda task_id, **_kwargs: status, + ) + monkeypatch.setattr( + von_routes, + "_get_current_chat_prompt_queue_scope", + lambda: { + "user_concept_id": "#V#test_user", + "organisation_concept_id": None, + "namespace": "#V#test_user", + }, ) response = app.test_client().get("/von/api/task/status/task-json-safe") @@ -76,8 +155,17 @@ def test_task_result_endpoint_jsonifies_generic_set_payloads(monkeypatch) -> Non monkeypatch.setattr( von_routes.background_task_registry, - "get_task_status", - lambda task_id: status, + "get_task_status_for_scope", + lambda task_id, **_kwargs: status, + ) + monkeypatch.setattr( + von_routes, + "_get_current_chat_prompt_queue_scope", + lambda: { + "user_concept_id": "#V#test_user", + "organisation_concept_id": None, + "namespace": "#V#test_user", + }, ) response = app.test_client().get("/von/api/task/result/task-json-safe") @@ -85,3 +173,41 @@ def test_task_result_endpoint_jsonifies_generic_set_payloads(monkeypatch) -> Non assert response.status_code == 200 body = response.get_json() assert body["result"]["allowed_write_tools"] == ["kb.write", "workflow.write"] + + +def test_task_cancellation_terminalises_its_still_queued_prompt(monkeypatch) -> None: + app = Flask(__name__) + app.register_blueprint(von_routes.von_bp, url_prefix="/von") + scope = { + "user_concept_id": "#V#test_user", + "organisation_concept_id": "#V#test_org", + "namespace": "#V#test_user@test_org", + } + status = SimpleNamespace(status="pending", queue_id="queue-1") + cancelled: list[tuple[dict[str, object], str]] = [] + + monkeypatch.setattr( + von_routes, + "_get_current_chat_prompt_queue_scope", + lambda: scope, + ) + monkeypatch.setattr( + von_routes, + "_get_background_task_status_for_scope", + lambda _task_id, _scope: status, + ) + monkeypatch.setattr( + von_routes.background_task_registry, + "request_cancellation_for_scope", + lambda *_args, **_kwargs: True, + ) + monkeypatch.setattr( + von_routes.chat_prompt_queue_service, + "cancel_prompt_record", + lambda *, scope, queue_id: cancelled.append((scope, queue_id)), + ) + + response = app.test_client().post("/von/api/task/cancel/task-1") + + assert response.status_code == 200 + assert cancelled == [(scope, "queue-1")] diff --git a/tests/backend/test_window_session_multi_org_isolation.py b/tests/backend/test_window_session_multi_org_isolation.py index 4237d933f..e872a15ff 100644 --- a/tests/backend/test_window_session_multi_org_isolation.py +++ b/tests/backend/test_window_session_multi_org_isolation.py @@ -89,8 +89,7 @@ def __init__(self, *args, **kwargs): "organisation_concept_id": organisation_concept_id, "role": ( "admin" - if organisation_concept_id - == "#V#university_of_auckland_strong_ai_lab" + if organisation_concept_id == "#V#university_of_auckland_strong_ai_lab" else "member" ), }, @@ -360,12 +359,12 @@ def find_one(self, query, projection=None): assert resp_home.status_code == 200 # Verify each window queried its own namespace - assert any( - "strong_ai_lab" in ns for ns in lab_namespaces - ), f"Lab should query lab namespace: {lab_namespaces}" - assert any( - "household" in ns for ns in home_namespaces - ), f"Home should query household namespace: {home_namespaces}" + assert any("strong_ai_lab" in ns for ns in lab_namespaces), ( + f"Lab should query lab namespace: {lab_namespaces}" + ) + assert any("household" in ns for ns in home_namespaces), ( + f"Home should query household namespace: {home_namespaces}" + ) class TestCreateChatSessionUsesWindowContext: @@ -459,8 +458,9 @@ def capture_create(*args, **kwargs): monkeypatch.setattr( chat_history_service, "has_chat_history_session", - lambda user_id, session_id, namespace=None: session_id - == payload["session_id"], + lambda user_id, session_id, namespace=None: ( + session_id == payload["session_id"] + ), ) import src.backend.services.shared_conversation_service as shared_conversation_service @@ -848,67 +848,106 @@ def test_cross_user_window_token_cannot_read_or_replace_org_scope(app_client): assert preserved.user_id == "#V#owner" assert preserved.organisation_concept_id == "secret_org" - def test_get_effective_context_falls_back_to_flask(self): - """get_effective_context should fall back to Flask session when no window session.""" - from src.backend.services.window_session_context_service import ( - get_effective_context, - ) - flask_session = { - "organisation_concept_id": "flask_org", - "namespace": "#V#user_1@flask_org", - "role_in_org": "member", - } +def test_get_effective_context_falls_back_to_flask(): + """No window selector retains the legacy Flask-session compatibility path.""" + from src.backend.services.window_session_context_service import ( + get_effective_context, + ) - # No window session - effective = get_effective_context(None, flask_session, "user_1") + flask_session = { + "organisation_concept_id": "flask_org", + "namespace": "#V#user_1@flask_org", + "role_in_org": "member", + } - assert effective["organisation_id"] == "flask_org" - assert effective["namespace"] == "#V#user_1@flask_org" - assert effective["role"] == "member" - assert effective["source"] == "flask_session" + effective = get_effective_context(None, flask_session, "user_1") - def test_get_effective_context_unknown_window_falls_back(self): - """get_effective_context should fall back when window session ID is unknown.""" - from src.backend.services.window_session_context_service import ( - get_effective_context, - ) + assert effective["organisation_id"] == "flask_org" + assert effective["namespace"] == "#V#user_1@flask_org" + assert effective["role"] == "member" + assert effective["source"] == "flask_session" - flask_session = { - "organisation_concept_id": "flask_org", - "namespace": "#V#user_1@flask_org", - "role_in_org": "member", - } - # Unknown window session ID - effective = get_effective_context("unknown_window_id", flask_session, "user_1") +def test_get_effective_context_unknown_window_falls_back_in_compatibility_mode(): + from src.backend.services.window_session_context_service import ( + get_effective_context, + ) - assert effective["organisation_id"] == "flask_org" - assert effective["source"] == "flask_session" + flask_session = { + "organisation_concept_id": "flask_org", + "namespace": "#V#user_1@flask_org", + "role_in_org": "member", + } - def test_get_effective_context_ignores_partial_window_scope(self): - """A partial window entry should not erase richer Flask org scope.""" - from src.backend.services.window_session_context_service import ( - get_effective_context, - get_window_session_store, + effective = get_effective_context("unknown_window_id", flask_session, "user_1") + + assert effective["organisation_id"] == "flask_org" + assert effective["source"] == "flask_session" + + +def test_get_effective_context_unknown_window_fails_closed_in_strict_mode(): + from src.backend.services.window_session_context_service import ( + WindowSessionContextUnavailable, + get_effective_context, + ) + + with pytest.raises(WindowSessionContextUnavailable): + get_effective_context( + "unknown_strict_window", + { + "organisation_concept_id": "wrong_flask_org", + "namespace": "#V#user_1@wrong_flask_org", + }, + "user_1", + require_known_window=True, ) - store = get_window_session_store() - partial_ctx = store.get_or_create("partial_window", user_id="user_1") - partial_ctx.chat_session_id = "chat_123" - store.set(partial_ctx) - flask_session = { - "organisation_concept_id": "flask_org", - "namespace": "#V#user_1@flask_org", - "role_in_org": "member", - "session_id": "flask_chat", - } +def test_get_effective_context_ignores_partial_window_scope_in_compatibility_mode(): + """A partial window entry should not erase richer Flask org scope.""" + from src.backend.services.window_session_context_service import ( + get_effective_context, + get_window_session_store, + ) + + store = get_window_session_store() + store.delete("partial_window") + partial_ctx = store.get_or_create("partial_window", user_id="user_1") + partial_ctx.chat_session_id = "chat_123" + store.set(partial_ctx) + + flask_session = { + "organisation_concept_id": "flask_org", + "namespace": "#V#user_1@flask_org", + "role_in_org": "member", + "session_id": "flask_chat", + } - effective = get_effective_context("partial_window", flask_session, "user_1") + effective = get_effective_context("partial_window", flask_session, "user_1") - assert effective["organisation_id"] == "flask_org" - assert effective["namespace"] == "#V#user_1@flask_org" - assert effective["role"] == "member" - assert effective["chat_session_id"] == "chat_123" - assert effective["source"] == "flask_session" + assert effective["organisation_id"] == "flask_org" + assert effective["namespace"] == "#V#user_1@flask_org" + assert effective["role"] == "member" + assert effective["chat_session_id"] == "chat_123" + assert effective["source"] == "flask_session" + + +def test_get_effective_context_partial_window_fails_closed_in_strict_mode(): + from src.backend.services.window_session_context_service import ( + WindowSessionContextUnavailable, + get_effective_context, + get_window_session_store, + ) + + store = get_window_session_store() + store.delete("strict_partial_window") + store.get_or_create("strict_partial_window", user_id="user_1") + + with pytest.raises(WindowSessionContextUnavailable): + get_effective_context( + "strict_partial_window", + {"namespace": "#V#user_1@wrong_org"}, + "user_1", + require_known_window=True, + ) diff --git a/tests/frontend/apiServiceWindowSessionIdentity.test.js b/tests/frontend/apiServiceWindowSessionIdentity.test.js new file mode 100644 index 000000000..691cd9fe5 --- /dev/null +++ b/tests/frontend/apiServiceWindowSessionIdentity.test.js @@ -0,0 +1,91 @@ +/** @jest-environment jsdom */ + +const apiServicePath = '../../src/frontend/web/von_interface/static/js/apiService.js'; + +function createBroadcastChannelHarness() { + const channelsByName = new Map(); + return class FakeBroadcastChannel { + constructor(name) { + this.name = name; + this.listeners = new Set(); + const channels = channelsByName.get(name) || new Set(); + channels.add(this); + channelsByName.set(name, channels); + } + + addEventListener(type, listener) { + if (type === 'message') this.listeners.add(listener); + } + + removeEventListener(type, listener) { + if (type === 'message') this.listeners.delete(listener); + } + + postMessage(data) { + queueMicrotask(() => { + for (const peer of channelsByName.get(this.name) || []) { + if (peer === this) continue; + for (const listener of peer.listeners) listener({ data }); + } + }); + } + + close() { + channelsByName.get(this.name)?.delete(this); + } + }; +} + +describe('apiService window-session request barrier', () => { + beforeEach(() => { + jest.resetModules(); + localStorage.clear(); + sessionStorage.clear(); + }); + + afterEach(() => { + delete global.BroadcastChannel; + delete window.BroadcastChannel; + delete global.fetch; + }); + + test('does not fetch until a copied tab ID rotates, then sends only the replacement header', async () => { + const FakeBroadcastChannel = createBroadcastChannelHarness(); + global.BroadcastChannel = FakeBroadcastChannel; + window.BroadcastChannel = FakeBroadcastChannel; + sessionStorage.setItem('von_window_session_id', 'ws_copied'); + + const incumbent = new FakeBroadcastChannel('von_window_session_coordination_v1'); + incumbent.addEventListener('message', ({ data }) => { + if (data?.type !== 'probe' || data.window_session_id !== 'ws_copied') return; + incumbent.postMessage({ + protocol: 'von_window_session_coordination.v1', + type: 'occupied', + window_session_id: 'ws_copied', + source_document_id: 'document-incumbent', + source_priority: '0:document-incumbent', + target_document_id: data.source_document_id, + probe_id: data.probe_id, + }); + }); + global.fetch = jest.fn(async () => ({ + ok: true, + json: async () => ({ success: true }), + })); + + const { getJson } = require(apiServicePath); + const request = getJson('/actor-scoped'); + + expect(global.fetch).not.toHaveBeenCalled(); + await expect(request).resolves.toEqual({ success: true }); + + expect(global.fetch).toHaveBeenCalledTimes(1); + const [, options] = global.fetch.mock.calls[0]; + expect(options.headers['X-Von-Window-Session']).toMatch(/^ws_/); + expect(options.headers['X-Von-Window-Session']).not.toBe('ws_copied'); + expect(sessionStorage.getItem('von_window_session_id')) + .toBe(options.headers['X-Von-Window-Session']); + + incumbent.close(); + }); +}); diff --git a/tests/frontend/chatTabAbort.test.js b/tests/frontend/chatTabAbort.test.js index 60feae90f..a4fe7e396 100644 --- a/tests/frontend/chatTabAbort.test.js +++ b/tests/frontend/chatTabAbort.test.js @@ -82,6 +82,7 @@ describe('formatChatTimestamp', () => { describe('chat abort behaviour', () => { beforeEach(() => { + __testOnly_resetChatRequestState(); document.body.innerHTML = `