Beats setup idempotency, Kibana dashboards and arm64 fix

molecule/elasticstack_default/molecule.yml

@@ -12,6 +12,7 @@ platforms:
       - beats
       - logstash
       - elasticsearch
+      - kibana
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian13}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
@@ -25,6 +26,7 @@ platforms:
       - beats
       - kibana
       - elasticsearch
+      - logstash
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian13}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:

molecule/elasticstack_default/prepare.yml

@@ -64,3 +64,20 @@
         state: latest
         update_cache: yes
       when: ansible_os_family == "Debian"
+
+    # Temporary CI workaround: containers cannot resolve each other by name
+    # (the runner's resolver is Azure's 168.63.129.16, which does not know the
+    # container hostnames). Seed /etc/hosts from gathered facts so name-based
+    # connections work. Remove once #524 (elasticstack_address_mode) lands and
+    # molecule sets elasticstack_address_mode: ip.
+    - name: Seed /etc/hosts from facts for inter-container name resolution
+      ansible.builtin.blockinfile:
+        path: /etc/hosts
+        marker: "# {mark} ANSIBLE CLUSTER HOSTS"
+        # /etc/hosts is a Docker bind-mount, so the default atomic temp+rename
+        # write fails with EBUSY; write in place instead.
+        unsafe_writes: true
+        block: |-
+          {% for h in groups['all'] %}
+          {{ hostvars[h].ansible_default_ipv4.address | default(hostvars[h].ansible_all_ipv4_addresses[0]) }} {{ hostvars[h].ansible_fqdn }} {{ hostvars[h].ansible_hostname }} {{ h }}
+          {% endfor %}

roles/beats/README.md

@@ -47,7 +47,7 @@ Run only parts of the role with `--tags`:
 | `beats_auditbeat` | `bool` | `false` | — | Install and manage Auditbeat. |
 | `beats_metricbeat` | `bool` | `false` | — | Install and manage Metricbeat. |
 | `beats_target_hosts` | `list` of `str` | `['localhost']` | — | Hosts the Beats ship to. Only used when the role runs standalone; with the other Elastic Stack roles the targets are determined automatically. |
-| `beats_fields` | `list` of `str` | N/A | — | Fields added to every input, given as a list of "key: value" strings (the global counterpart to the per-input fields). Unset by default. See the Filebeat inputs documentation. |
+| `beats_fields` | `list` of `str` | N/A | — | Global fields added to the log and syslog (tcp/udp) inputs (not to the mysql, journald or docker inputs), given as a list of "key: value" strings — the global counterpart to the per-input fields. Unset by default. See the Filebeat inputs documentation. |
 | `beats_logging` | `str` | `"file"` | — | Where the Beats log. Set to "file" to log into beats_logpath; any other value leaves the Beats built-in logging. |
 | `beats_loglevel` | `str` | `"info"` | — | Log level for all Beats. |
 | `beats_logpath` | `str` | `"/var/log/beats"` | — | Directory for the log files when beats_logging is "file". |
@@ -69,18 +69,18 @@ Run only parts of the role with `--tags`:
 | `beats_filebeat_modules` | `list` of `str` | N/A | — | List of Filebeat modules to enable (experimental). Unset by default. |
 | `beats_auditbeat_enable` | `bool` | `true` | — | Start and enable the Auditbeat service. |
 | `beats_auditbeat_output` | `str` | `"elasticsearch"` | `logstash`, `elasticsearch` | Where Auditbeat sends its events. |
-| `beats_auditbeat_setup` | `bool` | `true` | — | Run the Auditbeat setup (index management and pipelines). Only effective with the elasticsearch output. |
+| `beats_auditbeat_setup` | `bool` | `true` | — | Run the Auditbeat setup (index management, ingest pipelines and Kibana dashboards). Only effective with the elasticsearch output; loading the dashboards additionally requires Kibana to be reachable. |
 | `beats_auditbeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Auditbeat Logstash output. |
 | `beats_metricbeat_enable` | `bool` | `true` | — | Start and enable the Metricbeat service. |
 | `beats_metricbeat_output` | `str` | `"elasticsearch"` | `logstash`, `elasticsearch` | Where Metricbeat sends its events. |
 | `beats_metricbeat_modules` | `list` of `str` | `['system']` | — | Metricbeat modules to enable. |
 | `beats_metricbeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Metricbeat Logstash output. |
 | `beats_security` | `bool` | `false` | — | Activate TLS for the connections to the targets. Works with the other roles and elasticstack_full_stack to create certificates automatically, or with the beats_tls_* variables for custom certificates. |
-| `beats_ca_dir` | `str` | N/A | — | Base directory for custom CA certificates and keys. Unset by default; the role fills it depending on the stack variant (/opt/ca standalone, or /etc/beats/certs in a full stack). |
-| `beats_tls_key` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.key"` | — | Path to the private key file for custom certificates. |
-| `beats_tls_cert` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.crt"` | — | Path to the certificate for custom certificates. |
-| `beats_tls_cacert` | `str` | `"{{ beats_ca_dir \| default('') }}/ca.crt"` | — | Path to the CA certificate for custom certificates. |
-| `beats_tls_key_passphrase` | `str` | `"BeatsChangeMe"` | — | Passphrase of the private key. |
+| `beats_ca_dir` | `str` | N/A | — | Base directory for the Beats certificate, key and CA file. The role fills it per variant (/opt/ca standalone, /etc/beats/certs in a full stack) if you did not set it. |
+| `beats_tls_key` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.key"` | — | Path to the Beats private key. Defaults under beats_ca_dir; set it to bring your own. |
+| `beats_tls_cert` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.crt"` | — | Path to the Beats certificate. Defaults under beats_ca_dir; set it to bring your own. |
+| `beats_tls_cacert` | `str` | `"{{ beats_ca_dir \| default('') }}/ca.crt"` | — | Path to the CA certificate Beats trusts. Defaults under beats_ca_dir; set it to bring your own. |
+| `beats_tls_key_passphrase` | `str` | `"BeatsChangeMe"` | — | Passphrase of the Beats private key. |
 | `beats_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. |
 | `beats_cert_expiration_buffer` | `str` | `"+30d"` | — | Renew the certificate when it would expire within this period. Uses the community.crypto check_period format (e.g. "+30d"), not a plain number. |
 | `beats_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Beats certificate. Alternatively run the playbook with the renew_beats_cert tag. |

roles/beats/meta/argument_specs.yml

@@ -53,9 +53,10 @@ argument_specs:
         type: list
         elements: str
         description: >-
-          Fields added to every input, given as a list of "key: value" strings
-          (the global counterpart to the per-input fields). Unset by default.
-          See the Filebeat inputs documentation.
+          Global fields added to the log and syslog (tcp/udp) inputs (not to the
+          mysql, journald or docker inputs), given as a list of "key: value"
+          strings — the global counterpart to the per-input fields. Unset by
+          default. See the Filebeat inputs documentation.
 
       # ----- Logging -----
       beats_logging:
@@ -187,7 +188,10 @@ argument_specs:
       beats_auditbeat_setup:
         type: bool
         default: true
-        description: Run the Auditbeat setup (index management and pipelines). Only effective with the elasticsearch output.
+        description: >-
+          Run the Auditbeat setup (index management, ingest pipelines and Kibana
+          dashboards). Only effective with the elasticsearch output; loading the
+          dashboards additionally requires Kibana to be reachable.
 
       beats_auditbeat_loadbalance:
         type: bool
@@ -232,29 +236,29 @@ argument_specs:
       beats_ca_dir:
         type: str
         description: >-
-          Base directory for custom CA certificates and keys. Unset by default; the
-          role fills it depending on the stack variant (/opt/ca standalone, or
-          /etc/beats/certs in a full stack).
+          Base directory for the Beats certificate, key and CA file. The role
+          fills it per variant (/opt/ca standalone, /etc/beats/certs in a full
+          stack) if you did not set it.
 
       beats_tls_key:
         type: str
         default: "{{ beats_ca_dir | default('') }}/{{ inventory_hostname }}-beats.key"
-        description: Path to the private key file for custom certificates.
+        description: Path to the Beats private key. Defaults under beats_ca_dir; set it to bring your own.
 
       beats_tls_cert:
         type: str
         default: "{{ beats_ca_dir | default('') }}/{{ inventory_hostname }}-beats.crt"
-        description: Path to the certificate for custom certificates.
+        description: Path to the Beats certificate. Defaults under beats_ca_dir; set it to bring your own.
 
       beats_tls_cacert:
         type: str
         default: "{{ beats_ca_dir | default('') }}/ca.crt"
-        description: Path to the CA certificate for custom certificates.
+        description: Path to the CA certificate Beats trusts. Defaults under beats_ca_dir; set it to bring your own.
 
       beats_tls_key_passphrase:
         type: str
         default: BeatsChangeMe
-        description: Passphrase of the private key.
+        description: Passphrase of the Beats private key.
 
       beats_cert_validity_period:
         type: int

roles/beats/tasks/auditbeat.yml

@@ -57,8 +57,8 @@
     - beats_configuration
 
 - name: Setup Auditbeat in Elasticsearch
-  ansible.builtin.command: >
-    /usr/bin/auditbeat setup --pipelines --index-management &&
+  ansible.builtin.shell: >
+    /usr/bin/auditbeat setup --pipelines --index-management --dashboards &&
     /usr/bin/auditbeat version > /etc/auditbeat/pipeline_created
   run_once: true
   args:

roles/beats/tasks/filebeat.yml

@@ -81,7 +81,7 @@
 
     - name: Enable Ingest Pipelines
       ansible.builtin.shell: >
-        /usr/bin/filebeat setup --pipelines &&
+        /usr/bin/filebeat setup --pipelines --dashboards &&
         /usr/bin/filebeat version > /etc/filebeat/{{ item }}_pipeline_created
       args:
         creates: "/etc/filebeat/{{ item }}_pipeline_created"

roles/beats/tasks/metricbeat.yml

@@ -60,14 +60,14 @@
   when: beats_metricbeat_modules is defined
 
 - name: Enable Ingest Pipelines
-  ansible.builtin.command: >
+  ansible.builtin.shell: >
     metricbeat setup &&
     metricbeat version > /etc/metricbeat/pipelines_created
   args:
     creates: "/etc/metricbeat/pipelines_created"
   notify:
     - Restart Metricbeat
-  with_items: "{{ beats_metricbeat_modules }}"
+  run_once: true
   when:
     - beats_metricbeat_modules is defined
     - beats_metricbeat_output == "elasticsearch"

roles/beats/templates/auditbeat.yml.j2

@@ -19,7 +19,9 @@ auditbeat.modules:
     - host    # General host information, e.g. uptime, IPs
     - login   # User logins, logouts, and system boots.
     - process # Started and stopped processes
-    - socket  # Opened and closed sockets
+{% if ansible_architecture == 'x86_64' %}
+    - socket  # Opened and closed sockets (metricset not available in the arm64/aarch64 auditbeat build)
+{% endif %}
     - user    # User information
   state.period: 12h
   user.detect_password_changes: true
@@ -29,6 +31,13 @@ auditbeat.modules:
 setup.template.settings:
   index.number_of_shards: 1
 setup.kibana:
+{% if elasticstack_full_stack | bool and groups[elasticstack_kibana_group_name] | default([]) | length > 0 %}
+  host: "http://{{ groups[elasticstack_kibana_group_name] | first }}:{{ elasticstack_kibana_port }}"
+{% if beats_security | bool %}
+  username: "elastic"
+  password: "{{ beats_writer_password.stdout }}"
+{% endif %}
+{% endif %}
 {% if beats_auditbeat_output == "elasticsearch" %}
 output.elasticsearch:
 {% if beats_security | bool %}

roles/beats/templates/filebeat.yml.j2

@@ -109,6 +109,13 @@ filebeat.config.modules:
 setup.template.settings:
   index.number_of_shards: 1
 setup.kibana:
+{% if elasticstack_full_stack | bool and groups[elasticstack_kibana_group_name] | default([]) | length > 0 %}
+  host: "http://{{ groups[elasticstack_kibana_group_name] | first }}:{{ elasticstack_kibana_port }}"
+{% if beats_security | bool %}
+  username: "elastic"
+  password: "{{ beats_writer_password.stdout }}"
+{% endif %}
+{% endif %}
 {% if beats_filebeat_output == "elasticsearch" %}
 output.elasticsearch:
 {% if beats_security | bool %}

roles/beats/templates/metricbeat.yml.j2

@@ -6,6 +6,13 @@ metricbeat.config.modules:
 setup.template.settings:
   index.number_of_shards: 1
 setup.kibana:
+{% if elasticstack_full_stack | bool and groups[elasticstack_kibana_group_name] | default([]) | length > 0 %}
+  host: "http://{{ groups[elasticstack_kibana_group_name] | first }}:{{ elasticstack_kibana_port }}"
+{% if beats_security | bool %}
+  username: "elastic"
+  password: "{{ beats_writer_password.stdout }}"
+{% endif %}
+{% endif %}
 {% if beats_metricbeat_output == "elasticsearch" %}
 output.elasticsearch:
 {% if beats_security | bool %}

roles/elasticstack/README.md

@@ -39,7 +39,7 @@ The role runs automatically the first time any component role is applied to a ho
 | `elasticstack_version` | `str` | N/A | — | Exact package version to install (e.g. "8.8.1"). Unset by default: the collection then installs the latest, or reuses the version already present on the CA host. |
 | `elasticstack_full_stack` | `bool` | `true` | — | Treat the hosts as one coordinated stack (roles read each other's hosts, share the CA and wire the connections). Set to false to run a role standalone. |
 | `elasticstack_security` | `bool` | `true` | — | Enable X-Pack security across the stack. Only effective with the elastic variant. |
-| `elasticstack_override_beats_tls` | `bool` | `false` | — | Override the automatically derived TLS setting for the Beats input. |
+| `elasticstack_override_beats_tls` | `bool` | `false` | — | When true, do not auto-enable TLS between Beats and Logstash in a full stack (also skips Beats certificate generation). |
 | `elasticstack_elasticsearch_group_name` | `str` | `"elasticsearch"` | — | Inventory group that holds the Elasticsearch hosts. |
 | `elasticstack_logstash_group_name` | `str` | `"logstash"` | — | Inventory group that holds the Logstash hosts. |
 | `elasticstack_kibana_group_name` | `str` | `"kibana"` | — | Inventory group that holds the Kibana hosts. |

roles/elasticstack/meta/argument_specs.yml

@@ -63,7 +63,7 @@ argument_specs:
       elasticstack_override_beats_tls:
         type: bool
         default: false
-        description: Override the automatically derived TLS setting for the Beats input.
+        description: When true, do not auto-enable TLS between Beats and Logstash in a full stack (also skips Beats certificate generation).
 
       # ----- Inventory host groups -----
       elasticstack_elasticsearch_group_name:

roles/kibana/tasks/main.yml

@@ -134,17 +134,38 @@
         port: 5601
         timeout: 300
       register: kibana_wait
+
+    # An open port only means Kibana is starting; /api/status returns 503 ("not
+    # ready yet") until it is fully up, so downstream consumers (e.g. Beats
+    # dashboard setup) must not race it. Poll unauthenticated: a ready Kibana
+    # answers 200 (anonymous status allowed) or 401 (auth required but server up)
+    # depending on status.allowAnonymous / the version — treat both as ready.
+    - name: Wait for Kibana API to be ready
+      ansible.builtin.uri:
+        url: "{{ 'https' if kibana_tls | bool else 'http' }}://localhost:{{ elasticstack_kibana_port }}/api/status"
+        method: GET
+        validate_certs: false
+        status_code: [200, 401, 503]
+      register: kibana_api
+      until: kibana_api.status in [200, 401]
+      retries: 60
+      delay: 5
+      changed_when: false
   rescue:
-    - name: Debug Kibana status on failure
+    - name: Show why the Kibana readiness check failed
       ansible.builtin.debug:
-        msg: "Kibana failed to start: {{ kibana_wait }}"
-    - name: Show Kibana logs
+        msg: >-
+          Kibana readiness failed.
+          API check: status={{ kibana_api.status | default('n/a') }},
+          msg={{ kibana_api.msg | default('n/a') }}.
+          Port opened after {{ kibana_wait.elapsed | default('n/a') }}s.
+    - name: Show Kibana warnings and errors
       ansible.builtin.command:
-        cmd: journalctl -u kibana -n 100
+        cmd: journalctl -u kibana -n 100 -p warning
       register: kibana_logs
       changed_when: false
       no_log: true
-    - name: Output Kibana logs
+    - name: Output Kibana warnings and errors
       ansible.builtin.debug:
         var: kibana_logs.stdout