Ansible etcd — Deploy Distributed Key-Value Store

Introduction

etcd is the distributed key-value store that underpins Kubernetes — storing all cluster state, configuration, and service discovery data. Ansible automates etcd cluster deployment: static bootstrapping, TLS mutual authentication, automated backups, member management, and health monitoring.

Deploy etcd Cluster

---
- name: Deploy etcd cluster
  hosts: etcd_nodes
  become: true
  vars:
    etcd_version: "3.5.15"
    etcd_data_dir: /var/lib/etcd
    etcd_client_port: 2379
    etcd_peer_port: 2380
    etcd_cluster_token: my-etcd-cluster
  tasks:
    - name: Create etcd user
      ansible.builtin.user:
        name: etcd
        system: true
        shell: /usr/sbin/nologin
        home: "{{ etcd_data_dir }}"

    - name: Download etcd
      ansible.builtin.get_url:
        url: "https://github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-amd64.tar.gz"
        dest: /tmp/etcd.tar.gz

    - name: Extract etcd
      ansible.builtin.unarchive:
        src: /tmp/etcd.tar.gz
        dest: /tmp/
        remote_src: true

    - name: Install binaries
      ansible.builtin.copy:
        src: "/tmp/etcd-v{{ etcd_version }}-linux-amd64/{{ item }}"
        dest: "/usr/local/bin/{{ item }}"
        remote_src: true
        mode: '0755'
      loop: [etcd, etcdctl, etcdutl]

    - name: Create data directory
      ansible.builtin.file:
        path: "{{ etcd_data_dir }}"
        state: directory
        owner: etcd
        mode: '0700'

    - name: Deploy etcd config
      ansible.builtin.template:
        src: etcd.conf.yml.j2
        dest: /etc/etcd/etcd.conf.yml
        owner: etcd
        mode: '0640'
      notify: restart etcd

    - name: Create systemd service
      ansible.builtin.copy:
        dest: /etc/systemd/system/etcd.service
        content: |
          [Unit]
          Description=etcd
          After=network-online.target
          [Service]
          User=etcd
          Type=notify
          ExecStart=/usr/local/bin/etcd --config-file /etc/etcd/etcd.conf.yml
          Restart=on-failure
          RestartSec=5
          LimitNOFILE=65536
          [Install]
          WantedBy=multi-user.target
        mode: '0644'
      notify:
        - daemon reload
        - restart etcd

    - name: Allow etcd through firewall
      ansible.posix.firewalld:
        port: "{{ item }}/tcp"
        permanent: true
        state: enabled
        immediate: true
      loop:
        - "{{ etcd_client_port }}"
        - "{{ etcd_peer_port }}"

    - name: Start etcd
      ansible.builtin.service:
        name: etcd
        state: started
        enabled: true

  handlers:
    - name: daemon reload
      ansible.builtin.systemd:
        daemon_reload: true
    - name: restart etcd
      ansible.builtin.service:
        name: etcd
        state: restarted

Config Template

# templates/etcd.conf.yml.j2
name: {{ inventory_hostname }}
data-dir: {{ etcd_data_dir }}

initial-advertise-peer-urls: https://{{ ansible_default_ipv4.address }}:{{ etcd_peer_port }}
listen-peer-urls: https://{{ ansible_default_ipv4.address }}:{{ etcd_peer_port }}
advertise-client-urls: https://{{ ansible_default_ipv4.address }}:{{ etcd_client_port }}
listen-client-urls: https://{{ ansible_default_ipv4.address }}:{{ etcd_client_port }},https://127.0.0.1:{{ etcd_client_port }}

initial-cluster-token: {{ etcd_cluster_token }}
initial-cluster: {% for host in groups['etcd_nodes'] %}{{ host }}=https://{{ hostvars[host].ansible_default_ipv4.address }}:{{ etcd_peer_port }}{% if not loop.last %},{% endif %}{% endfor %}

initial-cluster-state: new

{% if etcd_tls_enabled | default(true) %}
client-transport-security:
  cert-file: /etc/etcd/ssl/server.crt
  key-file: /etc/etcd/ssl/server.key
  client-cert-auth: true
  trusted-ca-file: /etc/etcd/ssl/ca.crt

peer-transport-security:
  cert-file: /etc/etcd/ssl/peer.crt
  key-file: /etc/etcd/ssl/peer.key
  client-cert-auth: true
  trusted-ca-file: /etc/etcd/ssl/ca.crt
{% endif %}

auto-compaction-retention: "8"
auto-compaction-mode: periodic
snapshot-count: 10000

TLS Certificates

- name: Deploy etcd TLS certificates
  ansible.builtin.copy:
    src: "{{ item.src }}"
    dest: "{{ item.dest }}"
    owner: etcd
    mode: "{{ item.mode }}"
  loop:
    - { src: "ssl/ca.crt", dest: "/etc/etcd/ssl/ca.crt", mode: "0644" }
    - { src: "ssl/{{ inventory_hostname }}.crt", dest: "/etc/etcd/ssl/server.crt", mode: "0644" }
    - { src: "ssl/{{ inventory_hostname }}.key", dest: "/etc/etcd/ssl/server.key", mode: "0600" }
    - { src: "ssl/{{ inventory_hostname }}-peer.crt", dest: "/etc/etcd/ssl/peer.crt", mode: "0644" }
    - { src: "ssl/{{ inventory_hostname }}-peer.key", dest: "/etc/etcd/ssl/peer.key", mode: "0600" }
  notify: restart etcd

Automated Backups

- name: Deploy etcd backup script
  ansible.builtin.copy:
    dest: /usr/local/bin/etcd-backup.sh
    content: |
      #!/bin/bash
      BACKUP_DIR=/var/backups/etcd
      RETENTION_DAYS=7
      TIMESTAMP=$(date +%Y%m%d-%H%M%S)
      
      mkdir -p $BACKUP_DIR
      
      ETCDCTL_API=3 etcdctl snapshot save \
        $BACKUP_DIR/snapshot-$TIMESTAMP.db \
        --endpoints=https://127.0.0.1:2379 \
        --cacert=/etc/etcd/ssl/ca.crt \
        --cert=/etc/etcd/ssl/server.crt \
        --key=/etc/etcd/ssl/server.key
      
      # Verify backup
      ETCDCTL_API=3 etcdctl snapshot status \
        $BACKUP_DIR/snapshot-$TIMESTAMP.db --write-out=table
      
      # Clean old backups
      find $BACKUP_DIR -name "snapshot-*.db" -mtime +$RETENTION_DAYS -delete
    mode: '0755'

- name: Schedule daily backup
  ansible.builtin.cron:
    name: "etcd backup"
    minute: "0"
    hour: "2"
    job: "/usr/local/bin/etcd-backup.sh >> /var/log/etcd-backup.log 2>&1"
    user: etcd

Health Check

- name: Check etcd cluster health
  ansible.builtin.command: >
    etcdctl endpoint health
    --endpoints=https://{{ ansible_default_ipv4.address }}:{{ etcd_client_port }}
    --cacert=/etc/etcd/ssl/ca.crt
    --cert=/etc/etcd/ssl/server.crt
    --key=/etc/etcd/ssl/server.key
  register: health
  changed_when: false
  environment:
    ETCDCTL_API: "3"

- name: Check member list
  ansible.builtin.command: >
    etcdctl member list
    --endpoints=https://127.0.0.1:{{ etcd_client_port }}
    --cacert=/etc/etcd/ssl/ca.crt
    --cert=/etc/etcd/ssl/server.crt
    --key=/etc/etcd/ssl/server.key
    --write-out=table
  register: members
  changed_when: false
  run_once: true
  environment:
    ETCDCTL_API: "3"

Restore from Backup

- name: Restore etcd from snapshot
  ansible.builtin.command: >
    etcdctl snapshot restore {{ backup_file }}
    --data-dir={{ etcd_data_dir }}-restored
    --name={{ inventory_hostname }}
    --initial-cluster={{ initial_cluster }}
    --initial-advertise-peer-urls=https://{{ ansible_default_ipv4.address }}:{{ etcd_peer_port }}
  environment:
    ETCDCTL_API: "3"
  when: restore_etcd | default(false)

Troubleshooting

Database Too Large

- name: Compact and defragment
  ansible.builtin.shell: |
    etcdctl compact $(etcdctl endpoint status --write-out=json | python3 -c "import sys,json; print(json.load(sys.stdin)[0]['Status']['header']['revision'])")
    etcdctl defrag
  environment:
    ETCDCTL_API: "3"
    ETCDCTL_ENDPOINTS: "https://127.0.0.1:{{ etcd_client_port }}"
    ETCDCTL_CACERT: "/etc/etcd/ssl/ca.crt"
    ETCDCTL_CERT: "/etc/etcd/ssl/server.crt"
    ETCDCTL_KEY: "/etc/etcd/ssl/server.key"
  run_once: true
  changed_when: true

Conclusion

etcd is the backbone of Kubernetes and many distributed systems. Ansible deploys clusters with static bootstrapping, configures mutual TLS, schedules automated backups, and monitors cluster health. Template the config from inventory — the initial-cluster list is built dynamically from the etcd_nodes group. Use automated snapshots and tested restore procedures to ensure data durability.

Ansible etcd — Deploy Distributed Key-Value Store

Introduction

Deploy etcd Cluster

Config Template

TLS Certificates

Automated Backups

Health Check

Restore from Backup

Troubleshooting

Database Too Large

Related Articles

Conclusion