Ansible + Grafana Loki — Deploy Log Aggregation

Introduction

Grafana Loki is a horizontally scalable log aggregation system inspired by Prometheus. Unlike Elasticsearch, Loki indexes only labels (not full text), making it significantly cheaper to operate at scale. Combined with Promtail for log collection and Grafana for visualization, it forms a lightweight but powerful logging stack.

Deploy Loki Server

---
- name: Deploy Grafana Loki
  hosts: loki_servers
  become: true
  vars:
    loki_version: "3.4.0"
    loki_data_dir: /var/lib/loki
    loki_http_port: 3100
    loki_retention_period: 720h  # 30 days
  tasks:
    - name: Create Loki user
      ansible.builtin.user:
        name: loki
        system: true
        shell: /usr/sbin/nologin
        home: "{{ loki_data_dir }}"
        create_home: false

    - name: Create directories
      ansible.builtin.file:
        path: "{{ item }}"
        state: directory
        owner: loki
        group: loki
        mode: "0755"
      loop:
        - "{{ loki_data_dir }}"
        - "{{ loki_data_dir }}/chunks"
        - "{{ loki_data_dir }}/boltdb-shipper-active"
        - "{{ loki_data_dir }}/boltdb-shipper-cache"
        - /etc/loki

    - name: Download Loki binary
      ansible.builtin.get_url:
        url: "https://github.com/grafana/loki/releases/download/v{{ loki_version }}/loki-linux-amd64.zip"
        dest: /tmp/loki.zip

    - name: Extract Loki
      ansible.builtin.unarchive:
        src: /tmp/loki.zip
        dest: /usr/local/bin/
        remote_src: true
        mode: "0755"

    - name: Deploy Loki configuration
      ansible.builtin.template:
        src: loki-config.yaml.j2
        dest: /etc/loki/config.yaml
        owner: loki
        group: loki
        mode: "0644"
      notify: restart loki

    - name: Create systemd service
      ansible.builtin.copy:
        dest: /etc/systemd/system/loki.service
        mode: "0644"
        content: |
          [Unit]
          Description=Grafana Loki
          After=network.target

          [Service]
          Type=simple
          User=loki
          ExecStart=/usr/local/bin/loki-linux-amd64 -config.file=/etc/loki/config.yaml
          Restart=on-failure
          RestartSec=5
          LimitNOFILE=65536

          [Install]
          WantedBy=multi-user.target
      notify:
        - reload systemd
        - restart loki

    - name: Enable and start Loki
      ansible.builtin.systemd:
        name: loki
        enabled: true
        state: started

  handlers:
    - name: reload systemd
      ansible.builtin.systemd:
        daemon_reload: true
    - name: restart loki
      ansible.builtin.systemd:
        name: loki
        state: restarted

Loki Configuration Template

# templates/loki-config.yaml.j2
auth_enabled: false

server:
  http_listen_port: {{ loki_http_port }}
  grpc_listen_port: 9096

common:
  path_prefix: {{ loki_data_dir }}
  storage:
    filesystem:
      chunks_directory: {{ loki_data_dir }}/chunks
      rules_directory: {{ loki_data_dir }}/rules
  replication_factor: 1
  ring:
    instance_addr: {{ ansible_default_ipv4.address }}
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2024-01-01
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

limits_config:
  retention_period: {{ loki_retention_period }}
  max_query_series: 5000
  max_query_parallelism: 32

compactor:
  working_directory: {{ loki_data_dir }}/compactor
  retention_enabled: true
  delete_request_store: filesystem

Deploy Promtail Agents

---
- name: Deploy Promtail log collection agents
  hosts: all
  become: true
  vars:
    promtail_version: "3.4.0"
    loki_url: "http://{{ groups['loki_servers'][0] }}:3100"
  tasks:
    - name: Download Promtail
      ansible.builtin.get_url:
        url: "https://github.com/grafana/loki/releases/download/v{{ promtail_version }}/promtail-linux-amd64.zip"
        dest: /tmp/promtail.zip

    - name: Extract Promtail
      ansible.builtin.unarchive:
        src: /tmp/promtail.zip
        dest: /usr/local/bin/
        remote_src: true
        mode: "0755"

    - name: Deploy Promtail configuration
      ansible.builtin.template:
        src: promtail-config.yaml.j2
        dest: /etc/promtail/config.yaml
        mode: "0644"
      notify: restart promtail

    - name: Create systemd service
      ansible.builtin.copy:
        dest: /etc/systemd/system/promtail.service
        mode: "0644"
        content: |
          [Unit]
          Description=Promtail Log Collector
          After=network.target

          [Service]
          Type=simple
          ExecStart=/usr/local/bin/promtail-linux-amd64 -config.file=/etc/promtail/config.yaml
          Restart=on-failure

          [Install]
          WantedBy=multi-user.target
      notify:
        - reload systemd
        - restart promtail

    - name: Enable and start Promtail
      ansible.builtin.systemd:
        name: promtail
        enabled: true
        state: started

  handlers:
    - name: reload systemd
      ansible.builtin.systemd:
        daemon_reload: true
    - name: restart promtail
      ansible.builtin.systemd:
        name: promtail
        state: restarted

Promtail Configuration

# templates/promtail-config.yaml.j2
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /var/lib/promtail/positions.yaml

clients:
  - url: {{ loki_url }}/loki/api/v1/push

scrape_configs:
  - job_name: syslog
    static_configs:
      - targets: [localhost]
        labels:
          job: syslog
          host: {{ inventory_hostname }}
          __path__: /var/log/syslog

  - job_name: auth
    static_configs:
      - targets: [localhost]
        labels:
          job: auth
          host: {{ inventory_hostname }}
          __path__: /var/log/auth.log

  - job_name: journal
    journal:
      max_age: 12h
      labels:
        job: systemd-journal
        host: {{ inventory_hostname }}
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: unit

{% if inventory_hostname in groups.get('webservers', []) %}
  - job_name: nginx
    static_configs:
      - targets: [localhost]
        labels:
          job: nginx
          host: {{ inventory_hostname }}
          __path__: /var/log/nginx/*.log
    pipeline_stages:
      - regex:
          expression: '^(?P<remote_addr>[\w.]+) .* "(?P<method>\w+) (?P<path>[^ ]+) .* (?P<status>\d+)'
      - labels:
          method:
          status:
{% endif %}

LogQL Query Examples

# View all logs from a host
{host="web-1"} | json

# Filter by level
{job="syslog"} |= "error"

# Rate of errors per host
sum by (host) (rate({job="nginx"} |= "500" [5m]))

# Top 10 paths by request count
topk(10, sum by (path) (count_over_time({job="nginx"} [1h])))

Troubleshooting

# Check Loki is ready
curl http://localhost:3100/ready

# Check Promtail targets
curl http://localhost:9080/targets

# Check Loki ingestion metrics
curl http://localhost:3100/metrics | grep loki_ingester

# Verify labels
curl http://localhost:3100/loki/api/v1/labels

Conclusion

Loki + Promtail + Grafana provides a cost-effective alternative to the ELK stack. Ansible automates the full deployment — Loki servers with retention policies, Promtail agents with per-role scrape configs, and Grafana data source configuration. The label-based indexing means you get fast queries at a fraction of the storage cost.

Ansible + Grafana Loki — Deploy Log Aggregation

Ansible + Grafana Loki — Deploy Log Aggregation

Introduction

Deploy Loki Server

Loki Configuration Template

Deploy Promtail Agents

Promtail Configuration

LogQL Query Examples

Troubleshooting

Related Articles

Conclusion