Ansible Grafana Loki Log Aggregation

Grafana Loki is a horizontally-scalable log aggregation system inspired by Prometheus. Unlike Elasticsearch, Loki indexes only metadata (labels) rather than full log content, making it dramatically cheaper to operate at scale.

Why Loki over Elasticsearch

Feature	Loki	Elasticsearch
Storage cost	Low (compressed chunks)	High (full-text index)
Resource usage	Minimal (no indexing)	Heavy (JVM, RAM)
Query language	LogQL (PromQL-like)	Lucene/KQL
Label-based filtering	Native	Via fields
Grafana integration	Native	Plugin

Architecture Overview

┌──────────────┐     ┌──────────────┐     ┌──────────────┐
│   Promtail   │────▶│     Loki     │◀────│   Grafana    │
│  (agent)     │     │  (storage)   │     │  (queries)   │
└──────────────┘     └──────────────┘     └──────────────┘
     ▲                      │
     │                      ▼
  log files           Object Storage
  journald            (S3/MinIO/GCS)
  docker

Loki Deployment Playbook

---
- name: Deploy Grafana Loki
  hosts: loki_servers
  become: true
  vars:
    loki_version: "3.1.0"
    loki_user: loki
    loki_group: loki
    loki_data_dir: /var/lib/loki
    loki_config_dir: /etc/loki
    loki_http_port: 3100
    loki_grpc_port: 9096
    loki_retention_days: 30
    loki_storage_type: filesystem  # or s3
    loki_s3_bucket: "{{ vault_loki_s3_bucket | default('') }}"
    loki_s3_region: "{{ vault_loki_s3_region | default('us-east-1') }}"

  tasks:
    - name: Create Loki system user
      ansible.builtin.user:
        name: "{{ loki_user }}"
        group: "{{ loki_group }}"
        system: true
        shell: /usr/sbin/nologin
        create_home: false

    - name: Create Loki directories
      ansible.builtin.file:
        path: "{{ item }}"
        state: directory
        owner: "{{ loki_user }}"
        group: "{{ loki_group }}"
        mode: "0755"
      loop:
        - "{{ loki_config_dir }}"
        - "{{ loki_data_dir }}"
        - "{{ loki_data_dir }}/chunks"
        - "{{ loki_data_dir }}/boltdb-shipper-active"
        - "{{ loki_data_dir }}/boltdb-shipper-cache"
        - "{{ loki_data_dir }}/rules"

    - name: Download Loki binary
      ansible.builtin.get_url:
        url: "https://github.com/grafana/loki/releases/download/v{{ loki_version }}/loki-linux-amd64.zip"
        dest: /tmp/loki.zip
        mode: "0644"

    - name: Extract Loki binary
      ansible.builtin.unarchive:
        src: /tmp/loki.zip
        dest: /usr/local/bin/
        remote_src: true
        mode: "0755"

    - name: Rename binary
      ansible.builtin.copy:
        src: /usr/local/bin/loki-linux-amd64
        dest: /usr/local/bin/loki
        remote_src: true
        mode: "0755"

    - name: Deploy Loki configuration
      ansible.builtin.template:
        src: loki-config.yaml.j2
        dest: "{{ loki_config_dir }}/loki.yaml"
        owner: "{{ loki_user }}"
        group: "{{ loki_group }}"
        mode: "0640"
      notify: Restart Loki

    - name: Deploy Loki systemd unit
      ansible.builtin.template:
        src: loki.service.j2
        dest: /etc/systemd/system/loki.service
        mode: "0644"
      notify:
        - Reload systemd
        - Restart Loki

    - name: Enable and start Loki
      ansible.builtin.systemd:
        name: loki
        enabled: true
        state: started
        daemon_reload: true

    - name: Wait for Loki to be ready
      ansible.builtin.uri:
        url: "http://localhost:{{ loki_http_port }}/ready"
        status_code: 200
      register: loki_ready
      until: loki_ready.status == 200
      retries: 30
      delay: 5

  handlers:
    - name: Reload systemd
      ansible.builtin.systemd:
        daemon_reload: true

    - name: Restart Loki
      ansible.builtin.systemd:
        name: loki
        state: restarted

Loki Configuration Template

templates/loki-config.yaml.j2:

auth_enabled: false

server:
  http_listen_port: {{ loki_http_port }}
  grpc_listen_port: {{ loki_grpc_port }}
  log_level: info

common:
  path_prefix: {{ loki_data_dir }}
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: "2024-01-01"
      store: tsdb
      object_store: {{ loki_storage_type }}
      schema: v13
      index:
        prefix: index_
        period: 24h

{% if loki_storage_type == 'filesystem' %}
storage_config:
  filesystem:
    directory: {{ loki_data_dir }}/chunks
  tsdb_shipper:
    active_index_directory: {{ loki_data_dir }}/boltdb-shipper-active
    cache_location: {{ loki_data_dir }}/boltdb-shipper-cache
{% elif loki_storage_type == 's3' %}
storage_config:
  aws:
    s3: s3://{{ loki_s3_region }}/{{ loki_s3_bucket }}
    bucketnames: {{ loki_s3_bucket }}
    region: {{ loki_s3_region }}
  tsdb_shipper:
    active_index_directory: {{ loki_data_dir }}/boltdb-shipper-active
    cache_location: {{ loki_data_dir }}/boltdb-shipper-cache
    shared_store: s3
{% endif %}

limits_config:
  retention_period: {{ loki_retention_days * 24 }}h
  ingestion_rate_mb: 16
  ingestion_burst_size_mb: 32
  max_streams_per_user: 10000
  max_query_series: 500

compactor:
  working_directory: {{ loki_data_dir }}/compactor
  compaction_interval: 10m
  retention_enabled: true
  retention_delete_delay: 2h
  delete_request_store: {{ loki_storage_type }}

ruler:
  storage:
    type: local
    local:
      directory: {{ loki_data_dir }}/rules
  rule_path: {{ loki_data_dir }}/rules-temp
  alertmanager_url: http://alertmanager:9093
  ring:
    kvstore:
      store: inmemory
  enable_api: true

analytics:
  reporting_enabled: false

Promtail Agent Deployment

---
- name: Deploy Promtail Log Agent
  hosts: all
  become: true
  vars:
    promtail_version: "3.1.0"
    promtail_config_dir: /etc/promtail
    loki_url: "http://{{ groups['loki_servers'][0] }}:3100/loki/api/v1/push"
    promtail_positions_file: /var/lib/promtail/positions.yaml

  tasks:
    - name: Create Promtail directories
      ansible.builtin.file:
        path: "{{ item }}"
        state: directory
        mode: "0755"
      loop:
        - "{{ promtail_config_dir }}"
        - /var/lib/promtail

    - name: Download Promtail
      ansible.builtin.get_url:
        url: "https://github.com/grafana/loki/releases/download/v{{ promtail_version }}/promtail-linux-amd64.zip"
        dest: /tmp/promtail.zip

    - name: Extract Promtail
      ansible.builtin.unarchive:
        src: /tmp/promtail.zip
        dest: /usr/local/bin/
        remote_src: true
        mode: "0755"

    - name: Rename Promtail binary
      ansible.builtin.copy:
        src: /usr/local/bin/promtail-linux-amd64
        dest: /usr/local/bin/promtail
        remote_src: true
        mode: "0755"

    - name: Deploy Promtail configuration
      ansible.builtin.template:
        src: promtail-config.yaml.j2
        dest: "{{ promtail_config_dir }}/promtail.yaml"
        mode: "0644"
      notify: Restart Promtail

    - name: Deploy Promtail systemd unit
      ansible.builtin.copy:
        content: |
          [Unit]
          Description=Promtail Log Agent
          After=network-online.target
          Wants=network-online.target

          [Service]
          Type=simple
          User=root
          ExecStart=/usr/local/bin/promtail -config.file={{ promtail_config_dir }}/promtail.yaml
          Restart=always
          RestartSec=10

          [Install]
          WantedBy=multi-user.target
        dest: /etc/systemd/system/promtail.service
        mode: "0644"
      notify:
        - Reload systemd
        - Restart Promtail

    - name: Enable and start Promtail
      ansible.builtin.systemd:
        name: promtail
        enabled: true
        state: started
        daemon_reload: true

  handlers:
    - name: Reload systemd
      ansible.builtin.systemd:
        daemon_reload: true

    - name: Restart Promtail
      ansible.builtin.systemd:
        name: promtail
        state: restarted

Promtail Configuration Template

templates/promtail-config.yaml.j2:

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: {{ promtail_positions_file }}

clients:
  - url: {{ loki_url }}
    tenant_id: {{ promtail_tenant_id | default('default') }}

scrape_configs:
  # System logs via journald
  - job_name: journal
    journal:
      max_age: 12h
      labels:
        job: systemd-journal
        host: {{ inventory_hostname }}
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: unit
      - source_labels: ['__journal_priority_keyword']
        target_label: level

  # Application log files
  - job_name: system
    static_configs:
      - targets:
          - localhost
        labels:
          job: varlogs
          host: {{ inventory_hostname }}
          environment: {{ environment | default('production') }}
          __path__: /var/log/*.log

  # Nginx access logs
  - job_name: nginx
    static_configs:
      - targets:
          - localhost
        labels:
          job: nginx
          host: {{ inventory_hostname }}
          __path__: /var/log/nginx/access.log
    pipeline_stages:
      - regex:
          expression: '^(?P<remote_addr>[\w\.]+) - (?P<remote_user>[^ ]*) \[(?P<time_local>[^\]]*)\] "(?P<method>\w+) (?P<request>[^ ]*) (?P<protocol>[^"]*)" (?P<status>\d+) (?P<body_bytes_sent>\d+)'
      - labels:
          method:
          status:
      - timestamp:
          source: time_local
          format: "02/Jan/2006:15:04:05 -0700"

{% if 'docker' in group_names | default([]) %}
  # Docker container logs
  - job_name: docker
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 5s
    relabel_configs:
      - source_labels: ['__meta_docker_container_name']
        target_label: container
      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
        target_label: service
{% endif %}

Loki Alerting Rules

- name: Deploy Loki alerting rules
  hosts: loki_servers
  become: true
  tasks:
    - name: Create alerting rules
      ansible.builtin.copy:
        content: |
          groups:
            - name: application-errors
              rules:
                - alert: HighErrorRate
                  expr: |
                    sum(rate({job="varlogs"} |= "ERROR" [5m])) by (host)
                    > 10
                  for: 5m
                  labels:
                    severity: critical
                  annotations:
                    summary: "High error rate on {{ '{{' }} $labels.host {{ '}}' }}"
                    description: "More than 10 errors/sec for 5 minutes"

                - alert: AuthenticationFailures
                  expr: |
                    sum(rate({job="systemd-journal", unit="sshd.service"}
                      |= "Failed password" [10m])) by (host)
                    > 5
                  for: 5m
                  labels:
                    severity: warning
                  annotations:
                    summary: "SSH brute force on {{ '{{' }} $labels.host {{ '}}' }}"

                - alert: DiskSpaceLogVolume
                  expr: |
                    sum(rate({job="varlogs"} [1h])) by (host)
                    * 3600 * 24 > 10737418240
                  for: 1h
                  labels:
                    severity: warning
                  annotations:
                    summary: "Projected log volume >10GB/day on {{ '{{' }} $labels.host {{ '}}' }}"
        dest: /var/lib/loki/rules/default/alerts.yaml
        owner: loki
        group: loki
        mode: "0640"
      notify: Restart Loki

Grafana Data Source Configuration

- name: Add Loki datasource to Grafana
  hosts: grafana_servers
  tasks:
    - name: Configure Loki datasource
      community.grafana.grafana_datasource:
        name: Loki
        grafana_url: "http://localhost:3000"
        grafana_user: admin
        grafana_password: "{{ vault_grafana_password }}"
        ds_type: loki
        ds_url: "http://{{ groups['loki_servers'][0] }}:3100"
        is_default: false
        additional_json_data:
          derivedFields:
            - datasourceUid: tempo
              matcherRegex: "traceID=(\\w+)"
              name: TraceID
              url: "$${__value.raw}"

Multi-Tenant Configuration

# For multi-tenant Loki deployment
loki_auth_enabled: true
loki_tenants:
  - name: production
    retention_days: 90
  - name: staging
    retention_days: 14
  - name: development
    retention_days: 7

Update Promtail to send tenant header:

clients:
  - url: "http://loki:3100/loki/api/v1/push"
    tenant_id: "{{ environment }}"

Troubleshooting

Problem	Cause	Solution
Promtail not shipping logs	Permission denied on log files	Run Promtail as root or add to adm group
"entry out of order" errors	Clock skew between hosts	Sync NTP, or enable `unordered_writes`
High memory usage in Loki	Too many active streams	Reduce label cardinality, increase limits
Queries timeout	Scanning too much data	Add label filters before line filters
Missing logs after restart	Positions file lost	Ensure positions file is on persistent storage

Conclusion

Loki provides a cost-effective, Kubernetes-native logging solution that pairs naturally with Prometheus and Grafana. Ansible automates the entire stack — from Loki server deployment to Promtail agent distribution across hundreds of nodes — keeping your observability infrastructure as code.