Ansible ELK Stack — Deploy Elasticsearch, Logstash

Introduction

The ELK stack (Elasticsearch, Logstash, Kibana) — now called the Elastic Stack — is the most popular open-source solution for centralized logging and observability. Ansible automates the entire deployment: Elasticsearch cluster, Logstash pipelines, Kibana dashboards, and Filebeat agents on every server. This guide covers single-node dev setups through multi-node production clusters.

Architecture

┌─────────┐     ┌─────────┐     ┌──────────────┐     ┌─────────┐
│ Servers  │────►│ Filebeat│────►│  Logstash    │────►│ Elastic │
│ (logs)   │     │ (agent) │     │  (pipeline)  │     │ search  │
└─────────┘     └─────────┘     └──────────────┘     └────┬────┘
                                                           │
                                                     ┌─────▼────┐
                                                     │  Kibana   │
                                                     │  (UI)     │
                                                     └──────────┘

Install Elasticsearch

---
- name: Deploy Elasticsearch cluster
  hosts: elasticsearch
  become: true
  vars:
    es_version: "8.15"
    es_cluster_name: production-logs
    es_heap_size: "4g"
  tasks:
    - name: Import Elastic GPG key
      ansible.builtin.rpm_key:
        key: https://artifacts.elastic.co/GPG-KEY-elasticsearch
        state: present
      when: ansible_os_family == 'RedHat'

    - name: Add Elastic repository (Debian)
      ansible.builtin.apt_repository:
        repo: "deb https://artifacts.elastic.co/packages/{{ es_version }}/apt stable main"
        state: present
        filename: elastic
      when: ansible_os_family == 'Debian'

    - name: Install Elasticsearch
      ansible.builtin.package:
        name: elasticsearch
        state: present

    - name: Configure Elasticsearch
      ansible.builtin.template:
        src: elasticsearch.yml.j2
        dest: /etc/elasticsearch/elasticsearch.yml
        mode: '0660'
        owner: root
        group: elasticsearch
      notify: restart elasticsearch

    - name: Configure JVM heap
      ansible.builtin.template:
        src: jvm.options.j2
        dest: /etc/elasticsearch/jvm.options.d/heap.options
        mode: '0644'
      notify: restart elasticsearch

    - name: Start Elasticsearch
      ansible.builtin.service:
        name: elasticsearch
        state: started
        enabled: true

    - name: Wait for Elasticsearch
      ansible.builtin.uri:
        url: "http://localhost:9200/_cluster/health"
        method: GET
        status_code: 200
      register: es_health
      until: es_health.status == 200
      retries: 30
      delay: 10

  handlers:
    - name: restart elasticsearch
      ansible.builtin.service:
        name: elasticsearch
        state: restarted

Elasticsearch Config Template

# templates/elasticsearch.yml.j2
cluster.name: {{ es_cluster_name }}
node.name: {{ inventory_hostname }}
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch

network.host: {{ ansible_default_ipv4.address }}
http.port: 9200
transport.port: 9300

discovery.seed_hosts:
{% for host in groups['elasticsearch'] %}
  - {{ hostvars[host].ansible_default_ipv4.address }}
{% endfor %}

{% if inventory_hostname == groups['elasticsearch'][0] %}
cluster.initial_master_nodes:
{% for host in groups['elasticsearch'] %}
  - {{ host }}
{% endfor %}
{% endif %}

# Security (Elasticsearch 8+)
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: elastic-certificates.p12

# templates/jvm.options.j2
-Xms{{ es_heap_size }}
-Xmx{{ es_heap_size }}

Install Logstash

---
- name: Deploy Logstash
  hosts: logstash
  become: true
  tasks:
    - name: Install Logstash
      ansible.builtin.package:
        name: logstash
        state: present

    - name: Deploy Logstash pipeline
      ansible.builtin.template:
        src: logstash-pipeline.conf.j2
        dest: /etc/logstash/conf.d/main.conf
        mode: '0644'
      notify: restart logstash

    - name: Start Logstash
      ansible.builtin.service:
        name: logstash
        state: started
        enabled: true

  handlers:
    - name: restart logstash
      ansible.builtin.service:
        name: logstash
        state: restarted

Logstash Pipeline

# templates/logstash-pipeline.conf.j2
input {
  beats {
    port => 5044
  }
}

filter {
  if [fields][type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:hostname} %{DATA:program}(?:\[%{POSINT:pid}\])?: %{GREEDYDATA:message}" }
      overwrite => [ "message" ]
    }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }

  if [fields][type] == "nginx" {
    grok {
      match => { "message" => '%{IPORHOST:remote_addr} - %{DATA:remote_user} \[%{HTTPDATE:time_local}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{DATA:http_referer}" "%{DATA:http_user_agent}"' }
    }
    mutate {
      convert => {
        "status" => "integer"
        "body_bytes_sent" => "integer"
      }
    }
    geoip {
      source => "remote_addr"
    }
  }

  # Add hostname
  mutate {
    add_field => { "[@metadata][target_index]" => "%{[fields][type]}-%{+YYYY.MM.dd}" }
  }
}

output {
  elasticsearch {
    hosts => [
{% for host in groups['elasticsearch'] %}
      "http://{{ hostvars[host].ansible_default_ipv4.address }}:9200"{{ ',' if not loop.last else '' }}
{% endfor %}
    ]
    index => "%{[@metadata][target_index]}"
    user => "elastic"
    password => "{{ vault_elastic_password }}"
  }
}

Install Kibana

---
- name: Deploy Kibana
  hosts: kibana
  become: true
  tasks:
    - name: Install Kibana
      ansible.builtin.package:
        name: kibana
        state: present

    - name: Configure Kibana
      ansible.builtin.template:
        src: kibana.yml.j2
        dest: /etc/kibana/kibana.yml
        mode: '0660'
      notify: restart kibana

    - name: Start Kibana
      ansible.builtin.service:
        name: kibana
        state: started
        enabled: true

  handlers:
    - name: restart kibana
      ansible.builtin.service:
        name: kibana
        state: restarted

# templates/kibana.yml.j2
server.host: "0.0.0.0"
server.port: 5601
server.name: "{{ inventory_hostname }}"

elasticsearch.hosts:
{% for host in groups['elasticsearch'] %}
  - "http://{{ hostvars[host].ansible_default_ipv4.address }}:9200"
{% endfor %}

elasticsearch.username: "kibana_system"
elasticsearch.password: "{{ vault_kibana_password }}"

Deploy Filebeat Agents

---
- name: Deploy Filebeat on all servers
  hosts: all
  become: true
  tasks:
    - name: Install Filebeat
      ansible.builtin.package:
        name: filebeat
        state: present

    - name: Configure Filebeat
      ansible.builtin.template:
        src: filebeat.yml.j2
        dest: /etc/filebeat/filebeat.yml
        mode: '0600'
      notify: restart filebeat

    - name: Enable system module
      ansible.builtin.command: filebeat modules enable system
      args:
        creates: /etc/filebeat/modules.d/system.yml

    - name: Enable nginx module (web servers only)
      ansible.builtin.command: filebeat modules enable nginx
      args:
        creates: /etc/filebeat/modules.d/nginx.yml
      when: "'webservers' in group_names"

    - name: Start Filebeat
      ansible.builtin.service:
        name: filebeat
        state: started
        enabled: true

  handlers:
    - name: restart filebeat
      ansible.builtin.service:
        name: filebeat
        state: restarted

# templates/filebeat.yml.j2
filebeat.inputs:
  - type: filestream
    id: syslog
    paths:
      - /var/log/syslog
      - /var/log/messages
    fields:
      type: syslog

{% if 'webservers' in group_names %}
  - type: filestream
    id: nginx-access
    paths:
      - /var/log/nginx/access.log
    fields:
      type: nginx

  - type: filestream
    id: nginx-error
    paths:
      - /var/log/nginx/error.log
    fields:
      type: nginx-error
{% endif %}

output.logstash:
  hosts:
{% for host in groups['logstash'] %}
    - "{{ hostvars[host].ansible_default_ipv4.address }}:5044"
{% endfor %}

logging.level: warning

Index Lifecycle Management

- name: Configure ILM policy
  ansible.builtin.uri:
    url: "http://{{ groups['elasticsearch'][0] }}:9200/_ilm/policy/logs-policy"
    method: PUT
    body_format: json
    body:
      policy:
        phases:
          hot:
            min_age: "0ms"
            actions:
              rollover:
                max_size: "50gb"
                max_age: "1d"
          warm:
            min_age: "7d"
            actions:
              shrink:
                number_of_shards: 1
              forcemerge:
                max_num_segments: 1
          cold:
            min_age: "30d"
            actions:
              freeze: {}
          delete:
            min_age: "90d"
            actions:
              delete: {}
    user: elastic
    password: "{{ vault_elastic_password }}"
    status_code: [200]
  delegate_to: localhost

Docker-Based Deployment

- name: Deploy ELK with Docker Compose
  hosts: elk_server
  become: true
  tasks:
    - name: Deploy ELK stack
      ansible.builtin.copy:
        dest: /opt/elk/docker-compose.yml
        content: |
          services:
            elasticsearch:
              image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0
              environment:
                - discovery.type=single-node
                - xpack.security.enabled=false
                - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
              ports:
                - "9200:9200"
              volumes:
                - esdata:/usr/share/elasticsearch/data

            logstash:
              image: docker.elastic.co/logstash/logstash:8.15.0
              ports:
                - "5044:5044"
              volumes:
                - ./pipeline:/usr/share/logstash/pipeline
              depends_on:
                - elasticsearch

            kibana:
              image: docker.elastic.co/kibana/kibana:8.15.0
              ports:
                - "5601:5601"
              depends_on:
                - elasticsearch

          volumes:
            esdata:

    - name: Start ELK stack
      community.docker.docker_compose_v2:
        project_src: /opt/elk
        state: present

Troubleshooting

Elasticsearch Won't Start

- name: Check Elasticsearch logs
  ansible.builtin.command: journalctl -u elasticsearch --no-pager -n 50
  register: es_logs
  changed_when: false

# Common fix: increase vm.max_map_count
- ansible.posix.sysctl:
    name: vm.max_map_count
    value: '262144'
    sysctl_set: true

Cluster Health Yellow/Red

- name: Check cluster health
  ansible.builtin.uri:
    url: "http://localhost:9200/_cluster/health?pretty"
  register: health

- ansible.builtin.debug:
    var: health.json

Conclusion

Ansible deploys the entire ELK stack from inventory — Elasticsearch cluster with proper discovery, Logstash pipelines with grok filters, Kibana connected to the cluster, and Filebeat agents on every server. Template all configurations from group variables so adding a node is just adding a host to inventory. For smaller setups, use Docker Compose; for production, use dedicated hosts with ILM policies for automatic index lifecycle management.