Ansible + Datadog — Deploy Monitoring Agents and APM

Introduction

Datadog is one of the most popular cloud monitoring platforms, providing infrastructure monitoring, APM (Application Performance Monitoring), log management, and custom metrics. Deploying and configuring Datadog agents manually across hundreds of servers is tedious — Ansible automates the entire process, from agent installation to integration configuration.

Install the Datadog Ansible Collection

ansible-galaxy collection install datadog.dd

Basic Agent Deployment

---
- name: Deploy Datadog Agent
  hosts: all
  become: true
  vars:
    datadog_api_key: "{{ vault_datadog_api_key }}"
    datadog_site: "datadoghq.com"  # or datadoghq.eu, us3.datadoghq.com
  roles:
    - role: datadog.dd.agent
      vars:
        datadog_agent_major_version: 7
        datadog_enabled: true
        datadog_config:
          tags:
            - "env:{{ env }}"
            - "team:{{ team }}"
            - "service:{{ service_name }}"
          log_level: info
          apm_config:
            enabled: true
          process_config:
            enabled: "true"
          logs_enabled: true

Custom Agent Configuration

---
- name: Configure Datadog with integrations
  hosts: webservers
  become: true
  roles:
    - role: datadog.dd.agent
      vars:
        datadog_api_key: "{{ vault_datadog_api_key }}"
        datadog_config:
          tags:
            - "env:production"
            - "role:webserver"
          apm_config:
            enabled: true
            max_traces_per_second: 200
          logs_enabled: true
          log_level: warn

        # Integration configurations
        datadog_checks:
          nginx:
            instances:
              - nginx_status_url: "http://localhost:8080/nginx_status"
                tags:
                  - "cluster:{{ cluster_name }}"

          http_check:
            instances:
              - name: "Website Health"
                url: "https://{{ domain }}/health"
                timeout: 5
                http_response_status_code: "200"
                tags:
                  - "service:{{ service_name }}"

          process:
            instances:
              - name: nginx
                search_string: ["nginx"]
                exact_match: false
              - name: gunicorn
                search_string: ["gunicorn"]

Log Collection

datadog_checks:
  # Collect Nginx access and error logs
  nginx:
    logs:
      - type: file
        path: /var/log/nginx/access.log
        service: nginx
        source: nginx
        sourcecategory: http_web_access
      - type: file
        path: /var/log/nginx/error.log
        service: nginx
        source: nginx
        sourcecategory: http_web_error
        log_processing_rules:
          - type: multi_line
            name: nginx_error_multiline
            pattern: '\d{4}/\d{2}/\d{2}'

  # Collect application logs
  python:
    logs:
      - type: file
        path: /var/log/myapp/*.log
        service: myapp
        source: python
        tags:
          - "env:production"

APM Tracing Setup

---
- name: Configure APM
  hosts: app_servers
  become: true
  tasks:
    - name: Deploy Datadog agent with APM
      ansible.builtin.include_role:
        name: datadog.dd.agent
      vars:
        datadog_config:
          apm_config:
            enabled: true
            env: production
            max_traces_per_second: 500
            receiver_port: 8126
            analyzed_spans:
              myapp|flask.request: 1
              myapp|django.request: 1

    - name: Install Python APM library
      ansible.builtin.pip:
        name: ddtrace
        state: present

    - name: Configure application with tracing
      ansible.builtin.template:
        src: datadog-apm.env.j2
        dest: /etc/myapp/datadog.env
        mode: "0600"
      notify: restart myapp

# templates/datadog-apm.env.j2
DD_TRACE_AGENT_URL=http://localhost:8126
DD_SERVICE={{ service_name }}
DD_ENV={{ env }}
DD_VERSION={{ app_version }}
DD_TRACE_SAMPLE_RATE=1.0
DD_LOGS_INJECTION=true

Custom Checks

- name: Deploy custom Datadog check
  hosts: all
  become: true
  tasks:
    - name: Create custom check script
      ansible.builtin.copy:
        dest: /etc/datadog-agent/checks.d/queue_depth.py
        mode: "0644"
        content: |
          from datadog_checks.base import AgentCheck

          class QueueDepthCheck(AgentCheck):
              def check(self, instance):
                  import subprocess
                  result = subprocess.run(
                      ['rabbitmqctl', 'list_queues', 'name', 'messages'],
                      capture_output=True, text=True
                  )
                  for line in result.stdout.strip().split('\n'):
                      parts = line.split('\t')
                      if len(parts) == 2:
                          queue, depth = parts
                          self.gauge(
                              'custom.rabbitmq.queue_depth',
                              int(depth),
                              tags=[f'queue:{queue}']
                          )

    - name: Configure custom check
      ansible.builtin.copy:
        dest: /etc/datadog-agent/conf.d/queue_depth.d/conf.yaml
        mode: "0644"
        content: |
          init_config:
          instances:
            - min_collection_interval: 30
      notify: restart datadog-agent

Monitor and Alert Configuration

- name: Create Datadog monitors via API
  hosts: localhost
  connection: local
  vars:
    datadog_api_key: "{{ vault_datadog_api_key }}"
    datadog_app_key: "{{ vault_datadog_app_key }}"
  tasks:
    - name: Create high CPU alert
      ansible.builtin.uri:
        url: "https://api.datadoghq.com/api/v1/monitor"
        method: POST
        headers:
          DD-API-KEY: "{{ datadog_api_key }}"
          DD-APPLICATION-KEY: "{{ datadog_app_key }}"
          Content-Type: "application/json"
        body_format: json
        body:
          name: "High CPU Usage on {{ '{{host.name}}' }}"
          type: metric alert
          query: "avg(last_5m):avg:system.cpu.user{env:production} by {host} > 85"
          message: |
            CPU usage is above 85% on {{ '{{host.name}}' }}.
            @slack-ops-alerts @pagerduty-infra
          tags:
            - "env:production"
            - "team:infra"
          options:
            thresholds:
              critical: 85
              warning: 70
            notify_no_data: true
            no_data_timeframe: 10
        status_code: 200

Multi-Environment Deployment

# group_vars/production/datadog.yml
datadog_api_key: "{{ vault_datadog_api_key_prod }}"
datadog_site: "datadoghq.com"
datadog_env: production
datadog_apm_sample_rate: 0.1  # 10% sampling in production

# group_vars/staging/datadog.yml
datadog_api_key: "{{ vault_datadog_api_key_staging }}"
datadog_site: "datadoghq.com"
datadog_env: staging
datadog_apm_sample_rate: 1.0  # 100% sampling in staging

Troubleshooting

# Check agent status
sudo datadog-agent status

# Check integration configs
sudo datadog-agent configcheck

# Verify APM connectivity
curl -s http://localhost:8126/info | python3 -m json.tool

# Check agent logs
sudo tail -f /var/log/datadog/agent.log

Conclusion

The Datadog Ansible collection makes it straightforward to deploy monitoring agents, configure integrations, set up log collection, and enable APM tracing across your entire infrastructure. Use Ansible Vault for API keys, group_vars for environment-specific config, and custom checks for application-specific metrics.