Refactor/deps and editor refresh

pyproject.toml

@@ -11,7 +11,7 @@ license = { file = "LICENSE.txt" }
 requires-python = ">=3.12"
 dependencies = [
     "cryptography>=46.0.6",
-    "django>=6.0.3",
+    "django>=6.0.8,<6.1",
     "django-browser-reload>=1.21.0",
     "django-htmx>=1.27.0",
     "django-login-required-middleware>=0.9.0",
@@ -57,6 +57,9 @@ otel = [
 
 [tool.uv]
 package = true
+override-dependencies = [
+    "sqlparse>=0.6.0",
+]
 
 [tool.hatch.build.targets.sdist]
 include = [
@@ -104,7 +107,7 @@ exclude = [
 [dependency-groups]
 dev = [
     "pre-commit>=4.5.1",
-    "pytest>=9.0.2",
+    "pytest>=9.0.3",
     "pytest-cov>=7.1.0",
     "pytest-django>=4.10.0",
     "testcontainers[postgres]>=4.15.0",

src/logstashui/PipelineManager/agent_modes.py

@@ -773,7 +773,7 @@ def list_simulation_targets(active_only: bool = True, *, ensure_embedded: bool =
             if embedded_probe_failed(conn):
                 continue
             discovered = is_embedded_discovered(conn)
-            # Closed select: terse; detail on hover / open option list
+            # Embedded is a fixed system target, so its display name stays concise.
             label = "embedded"
             ver_label = version or "docker"
             agent_port = conn.agent_api_port or EMBEDDED_AGENT_API_PORT
@@ -795,8 +795,8 @@ def list_simulation_targets(active_only: bool = True, *, ensure_embedded: bool =
             n = conn.instance_id or "?"
             ver_label = version or "system"
             host = conn.host or "127.0.0.1"
-            label = f"simulate-{n}"
-            detail = f"simulate-{n} · {host} · Logstash {ver_label}"
+            label = conn.name
+            detail = f"{conn.name} · {host} · Logstash {ver_label}"
             agent_port = conn.agent_api_port
             if agent_port is None and conn.instance_id:
                 agent_port = SIMULATE_AGENT_API_BASE + conn.instance_id

src/logstashui/PipelineManager/simulation.py

@@ -116,7 +116,7 @@ def SelectSimulationTarget(request):
 
     Examples:
         payload = {"connection_id": 12}
-        # {"success": True, "selected_connection_id": 12, "label": "simulate-1", ...}
+        # {"success": True, "selected_connection_id": 12, "label": "LogstashUI-simulate-1", ...}
     """
     if request.method != "POST":
         return JsonResponse({"error": "Method not allowed"}, status=405)

src/logstashui/PipelineManager/static/css/move_mode.css

@@ -137,3 +137,8 @@
     content: 'Wrong section';
     color: #ef4444;
 }
+
+/* Invalid positions above elastic_integration are omitted entirely. */
+.move-mode .insertion-point.pinned-order-hidden {
+    display: none !important;
+}

src/logstashui/PipelineManager/static/js/move_mode.js

@@ -210,16 +210,37 @@ function updateInsertionPointsForMoveMode() {
     const isMovingCondition = movingComponent && movingComponent.plugin === 'if';
 
     insertionPoints.forEach(point => {
+        point.classList.remove('pinned-order-hidden');
+
         // Determine the section type of this insertion point
         const container = point.closest('[data-type]');
         const sectionType = container ? container.dataset.type : null;
+        const targetLocation = getInsertionPointLocation(point);
+        const pinnedCount = window.getPinnedFilterCount?.() || 0;
+        const movingPinned = window.isPinnedPipelineComponent?.(movingComponent);
+
+        // Pinned integrations can only be reordered inside their top-level prefix.
+        if (movingPinned && (!targetLocation || targetLocation.isNested ||
+            targetLocation.type !== 'filter' || targetLocation.index > pinnedCount)) {
+            point.classList.add('pinned-order-hidden', 'disabled');
+            return;
+        }
 
         // Disable insertion points in different sections
         if (sectionType && sectionType !== window.moveMode.componentType) {
             point.classList.add('disabled');
             return;
         }
 
+        // A regular filter component cannot be placed before the pinned prefix.
+        if (pinnedCount > 0 &&
+            !movingPinned &&
+            targetLocation && !targetLocation.isNested &&
+            targetLocation.type === 'filter' && targetLocation.index < pinnedCount) {
+            point.classList.add('pinned-order-hidden', 'disabled');
+            return;
+        }
+
         // If moving a condition, disable insertion points inside itself
         if (isMovingCondition) {
             const componentContainer = point.closest('.component-container');
@@ -259,6 +280,19 @@ function dropComponentAtInsertionPoint(insertionPoint) {
         return;
     }
 
+    const movingComponent = findComponentById(window.moveMode.componentId);
+    const pinnedCount = window.getPinnedFilterCount?.() || 0;
+    const movingPinned = window.isPinnedPipelineComponent?.(movingComponent);
+    if (movingPinned && (targetLocation.isNested ||
+        targetLocation.type !== 'filter' || targetLocation.index > pinnedCount)) {
+        return;
+    }
+    if (movingComponent && !movingPinned &&
+        !targetLocation.isNested && targetLocation.type === 'filter' &&
+        targetLocation.index < pinnedCount) {
+        return;
+    }
+
     // Validate that we're not dropping in the same location
     if (isSameLocation(window.moveMode.sourceLocation, targetLocation)) {
         deactivateMoveMode();
@@ -411,6 +445,7 @@ function moveComponent(source, target) {
 
     // Insert at target location
     insertComponentAtLocation(component, target, adjustedTargetIndex);
+    window.enforcePinnedPipelineOrder?.();
 
     // Refresh the UI
     newlyAddedPluginId = component.id;

src/logstashui/PipelineManager/static/js/pipeline_editor.js

@@ -13,6 +13,59 @@ let isSelectionMode = false;
 
 // Note: moveMode is now defined in move_mode.js as window.moveMode
 
+const PINNED_FILTER_PLUGIN = 'elastic_integration';
+
+window.isPinnedPipelineComponent = function isPinnedPipelineComponent(component) {
+    return Boolean(component && component.plugin === PINNED_FILTER_PLUGIN);
+};
+
+function extractPinnedPipelineComponents(componentList, pinnedComponents) {
+    if (!Array.isArray(componentList)) return [];
+
+    return componentList.filter(component => {
+        if (window.isPinnedPipelineComponent(component)) {
+            pinnedComponents.push(component);
+            return false;
+        }
+
+        if (component && component.plugin === 'if' && component.config) {
+            component.config.plugins = extractPinnedPipelineComponents(
+                component.config.plugins,
+                pinnedComponents
+            );
+            (component.config.else_ifs || []).forEach(elseIf => {
+                elseIf.plugins = extractPinnedPipelineComponents(
+                    elseIf.plugins,
+                    pinnedComponents
+                );
+            });
+            if (component.config.else) {
+                component.config.else.plugins = extractPinnedPipelineComponents(
+                    component.config.else.plugins,
+                    pinnedComponents
+                );
+            }
+        }
+        return true;
+    });
+}
+
+window.enforcePinnedPipelineOrder = function enforcePinnedPipelineOrder() {
+    const pinnedComponents = [];
+    ['input', 'filter', 'output'].forEach(type => {
+        if (!Array.isArray(components[type])) return;
+        components[type] = extractPinnedPipelineComponents(
+            components[type],
+            pinnedComponents
+        );
+    });
+    components.filter = pinnedComponents.concat(components.filter || []);
+};
+
+window.getPinnedFilterCount = function getPinnedFilterCount() {
+    return (components.filter || []).filter(window.isPinnedPipelineComponent).length;
+};
+
 /**
  * Trigger pipeline warming / slot preallocation.
  * Used on page load/refresh, after plugin changes, and (as fallback) when the
@@ -1605,11 +1658,16 @@ function updateBlockingProblemsIndicator() {
 }
 
 function loadExistingComponents() {
+    // elastic_integration must remain a top-level prefix of the Filter pipeline.
+    window.enforcePinnedPipelineOrder();
+
     // Check if we're in simulation mode before clearing
     const wasInSimulationMode = document.querySelector('.simulation-executed-badge') !== null;
     const simulationNodes = wasInSimulationMode && window.simulationData ? window.simulationData.nodes : null;
-    const originalEventData = wasInSimulationMode && window.simulationResultsCache ?
-        Object.values(window.simulationResultsCache)[0]?.originalEvent : null;
+    const originalEventData = wasInSimulationMode && window.simulationData ?
+        window.simulationData.originalEvent : null;
+    const outputEventData = wasInSimulationMode && window.simulationData ?
+        window.simulationData.outputEvent : null;
 
     // Clears all existing components first
     const componentTypes = ['input', 'filter', 'output'];
@@ -1670,7 +1728,7 @@ function loadExistingComponents() {
 
     // Restore simulation data if we were in simulation mode
     if (wasInSimulationMode && simulationNodes && typeof markExecutedPlugins === 'function') {
-        markExecutedPlugins(simulationNodes, originalEventData);
+        markExecutedPlugins(simulationNodes, originalEventData, outputEventData);
     }
 
     // If in graph mode, re-render the graph
@@ -1955,12 +2013,13 @@ function createComponentElement(component, depth = 0, isConditional = false, par
 
 // Check if this is a comment plugin - apply special styling
     const isComment = component.plugin === 'comment';
+    const isPinnedPlugin = window.isPinnedPipelineComponent(component);
 
 // Alternate background colors based on depth
     const bgColor = isComment ? 'bg-gray-800' : (depth % 2 === 0 ? 'bg-gray-700' : 'bg-gray-600');
     const el = document.createElement('div');
     const commentClass = isComment ? 'comment-plugin' : '';
-    el.className = `${bgColor} p-3 rounded mb-2 relative group draggable-item ${commentClass}`;
+    el.className = `${bgColor} p-3 rounded mb-2 relative group draggable-item ${commentClass} ${isPinnedPlugin ? 'pinned-component' : ''}`;
     el.dataset.id = component.id;
 
 // Get plugin info for description and type
@@ -2018,8 +2077,17 @@ function createComponentElement(component, depth = 0, isConditional = false, par
     // Validate required fields
     const validation = validateRequiredFields(component);
 
+    const pinIconHtml = isPinnedPlugin ? `
+      <span class="ml-1 inline-flex text-blue-300" title="Pinned to the top of the Filter pipeline" aria-label="Pinned to top">
+        <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24" aria-hidden="true">
+          <path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.8" d="M8 3h8l-1 6 3 3H6l3-3-1-6ZM12 12v9" />
+        </svg>
+      </span>
+    ` : '';
+
     el.innerHTML = `
-<button class="move-handle" data-component-id="${component.id}" title="Click to move this component">
+<button class="move-handle" data-component-id="${component.id}"
+        title="${isPinnedPlugin ? 'Reorder pinned elastic integrations' : 'Click to move this component'}">
   <svg fill="none" stroke="currentColor" viewBox="0 0 24 24">
     <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 8h16M4 16h16" />
   </svg>
@@ -2029,6 +2097,7 @@ function createComponentElement(component, depth = 0, isConditional = false, par
     <div class="flex items-center ${isComment ? 'flex-wrap' : ''}">
       ${imageHtml}
       <span class="font-medium text-white">${component.plugin}</span>
+      ${pinIconHtml}
       <span class="ml-2 px-1.5 py-0.5 text-xs rounded-full ${typeColor}">
         ${component.type.charAt(0).toUpperCase() + component.type.slice(1)}
       </span>
@@ -3342,8 +3411,14 @@ function addElseToConditional(componentId) {
 
 
 // Function to delete an else-if block
-function deleteElseIfBlock(componentId, elseIfIndex) {
-    if (!confirm('Are you sure you want to remove this else-if block and all its plugins?')) {
+async function deleteElseIfBlock(componentId, elseIfIndex) {
+    const confirmed = await ConfirmationModal.show(
+        'Are you sure you want to remove this else-if block and all its plugins?',
+        'Remove Else-If Block',
+        'Remove'
+    );
+
+    if (!confirmed) {
         return;
     }
 
@@ -3369,8 +3444,14 @@ function deleteElseIfBlock(componentId, elseIfIndex) {
 }
 
 // Function to delete an else block
-function deleteElseBlock(componentId) {
-    if (!confirm('Are you sure you want to remove this else block and all its plugins?')) {
+async function deleteElseBlock(componentId) {
+    const confirmed = await ConfirmationModal.show(
+        'Are you sure you want to remove this else block and all its plugins?',
+        'Remove Else Block',
+        'Remove'
+    );
+
+    if (!confirmed) {
         return;
     }
 
@@ -3939,6 +4020,21 @@ window.toggleFilePathInput = function(inputId, isIgnored) {
     }
 };
 
+/**
+ * Independently collapse or expand one of the main pipeline sections.
+ */
+window.togglePipelineSection = function togglePipelineSection(sectionId) {
+    const section = document.getElementById(sectionId);
+    const header = section && section.querySelector('.pipeline-section-header');
+    const contentId = header && header.getAttribute('aria-controls');
+    const content = contentId && document.getElementById(contentId);
+    if (!section || !header || !content) return;
+
+    const willCollapse = header.getAttribute('aria-expanded') === 'true';
+    header.setAttribute('aria-expanded', String(!willCollapse));
+    content.classList.toggle('hidden', willCollapse);
+};
+
 // Browse file path for simulation and upload the file
 window.browseFilePathForSimulation = function(inputId) {
     // Create a hidden file input

src/logstashui/PipelineManager/static/js/sim_targets.js

@@ -127,7 +127,7 @@
     targets.forEach((t) => {
       const opt = document.createElement('option');
       opt.value = String(t.connection_id);
-      // Closed control shows terse label; open list + title show host/version
+      // The control shows the full agent name; its title adds host/version details.
       opt.textContent = shortLabel(t);
       opt.dataset.detail = detailLabel(t);
       opt.dataset.policyType = t.policy_type || '';

src/logstashui/PipelineManager/static/js/simulation_modal.js

@@ -1048,15 +1048,6 @@ function createSimulationOverlay(runId, slotId, filterCount) {
           </div>
         </div>
         <div class="flex items-center gap-2">
-          <button onclick="viewSimulationLogs()" 
-                  id="viewLogsBtn"
-                  class="text-blue-400 hover:text-blue-300 px-3 py-1 rounded hover:bg-gray-700 flex items-center gap-2 text-sm"
-                  title="View Logstash logs for this simulation">
-            <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
-              <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"></path>
-            </svg>
-            View Logs
-          </button>
           <button onclick="toggleOverlayExpand()" 
                   id="expandOverlayBtn"
                   class="text-gray-400 hover:text-white px-2 py-1 rounded hover:bg-gray-700"
@@ -1141,4 +1132,4 @@ if (modal) {
 window.addEventListener('beforeunload', clearSimulationResults);
 document.addEventListener('visibilitychange', function() {
 if (document.hidden) clearSimulationResults();
-});
+});