Ansible j2模板引用eth1 IPv4配置HAProxy报错排查

部署背景 #

• 基于VirtualBox平台通过Vagrant部署3台虚拟机节点：2台运行Apache的Web节点、1台运行HAProxy的负载均衡节点
• 节点启动后自动触发Ansible Playbook执行：先完成两台Web节点的Apache服务部署配置，待Web节点就绪后再配置HAProxy负载均衡

故障现象 #

• 故障1：渲染HAProxy配置的Jinja2模板时，无法通过hostvars内置变量获取两台Web服务器eth1网卡的IPv4地址，Playbook执行到Configuring haproxy任务时报错：AnsibleUndefinedVariable: 'ansible.vars.hostvars.HostVarsVars object' has no attribute 'ansible_eth1'
• 故障2：HAProxy部署完成后，访问10.2.2.20:8080端口无响应，Chrome浏览器返回ERR_CONNECTION_REFUSED错误

现有环境配置 #

1. Vagrantfile配置 #

# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
 (1..2).each do |i|
  config.vm.define "HDVLD-TEST-WEB0#{i}" do |webserver|
    webserver.vm.box = "ubuntu/trusty64"
    webserver.vm.hostname = "HDVLD-TEST-WEB0#{i}"
    webserver.vm.network :private_network, ip: "10.2.2.1#{i}"
    webserver.vm.provider :virtualbox do |vb|
      vb.memory = "524"
      vb.customize ["modifyvm", :id, "--nested-hw-virt", "on"]
    end
    webserver.vm.provision "shell" do |shell|
      ssh_pub_key = File.readlines("#{Dir.home}/.ssh/id_rsa.pub")
      shell.inline = <<-SHELL
      echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys
      echo #{ssh_pub_key} >> /root/.ssh/authorized_keys
    SHELL
    end
  end
  config.vm.define "HDVLD-TEST-LB01" do |lb_server|
    lb_server.vm.box = "ubuntu/trusty64"
    lb_server.vm.hostname = "HDVLD-TEST-LB01"
    lb_server.vm.network :private_network, ip: "10.2.2.20"
    lb_server.vm.provider :virtualbox do |vb|
      vb.memory = "524"
      vb.customize ["modifyvm", :id, "--nested-hw-virt", "on"]
  end
    lb_server.vm.provision "shell" do |shell|
      ssh_pub_key = File.readlines("#{Dir.home}/.ssh/id_rsa.pub")
      shell.inline = <<-SHELL
      echo #{ssh_pub_key} >> /home/vagrant/.ssh/authorized_keys
      echo #{ssh_pub_key} >> /root/.ssh/authorized_keys
    SHELL
    end
  end
  config.vm.provision :ansible do |ansible|
    ansible.playbook = "webserver_test.yml"
    ansible.groups = {
      "webservers" => ["HDVLD-TEST-WEB01", "HDVLD-TEST-WEB02"],
      "loadbalancer" => ["HDVLD-TEST-LB01"]
    }
  end
end
end

2. Playbook配置（webserver_test.yml） #

- hosts: webservers
  become: true
  vars_files: vars/default.yml
  gather_facts: True
  tasks:
    - name: Gather facts from new server
      delegate_facts: True
      setup:
        filter: ansible_eth1.ipv4.address
    - name: Debug facts from Server
      delegate_facts: True
      debug:
        var: ansible_eth1.ipv4.address
    - name: UPurge
      apt: purge=yes
    - name: Install latest version of Apache
      apt: name=apache2 update_cache=yes state=latest
    - name: Install latest version of Facter
      apt: name=facter state=latest
    - name: Create document root for your domain
      file:
        path: /var/www/{{ http_host }}
        state: directory
        mode: '0755'
    - name: Copy your index page
      template:
        src: "files/index.html.j2"
        dest: "/var/www/{{ http_host }}/index.html"
    - name: Set up virtuahHost
      template:
        src: "files/apache.conf.j2"
        dest: "/etc/apache2/sites-available/{{ http_conf }}"
      notify: restart-apache
    - name: Enable new site {{ http_host }}
      command: a2ensite {{ http_host }}
    - name: Disable default site
      command: a2dissite 000-default
      when: disable_default
      notify: restart-apache
    - name: "UFW firewall allow HTTP on port {{ http_port }}"
      ufw:
        rule: allow
        port: "{{ http_port }}"
        proto: tcp
  handlers:
    - name: restart-apache
      service:
        name: apache2
        state: restarted
- hosts: loadbalancer
  become: true
  vars_files: vars/default.yml
  gather_facts: true
  tasks:
    - name: "Installing haproxy"
      package:
        name: "haproxy"
        state: present
    - name: "Starting haproxy"
      service:
        name: "haproxy"
        state: started
        enabled: yes
    - name: "Configuring haproxy"
      template:
        src: "files/haproxy.conf.j2"
        dest: "/etc/haproxy/haproxy.cfg"
      notify: restart-haproxy
    - name: "UFW firewall allow Proxy on port {{ proxy_port }}"
      ufw:
        rule: allow
        port: "{{ proxy_port }}"
        proto: tcp
    - name: "UFW firewall allow static port on port {{ staticlb_port }}"
      ufw:
        rule: allow
        port: "{{ staticlb_port }}"
        proto: tcp
    - name: Gather facts from new Server
      setup:
        filter: ansible_default_ipv4.address
  handlers:
    - name: restart-haproxy
      service:
        name: haproxy
        state: restarted

3. HAProxy配置模板（haproxy.conf.j2） #

global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     1000
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats
    ssl-default-bind-ciphers PROFILE=SYSTEM
    ssl-default-server-ciphers PROFILE=SYSTEM
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
    listen haproxy-monitoring *:{{ proxy_port }}
frontend main
    bind *:{{ http_port }}
    acl url_static       path_beg       -i /static /images /javascript /stylesheets
    acl url_static       path_end       -i .jpg .gif .png .css .js
    use_backend static          if url_static
    default_backend             app
backend static
    balance     roundrobin
    server      static 127.0.0.1:{{ staticlb_port }} check
backend app
    balance     roundrobin
    {% for host in groups['webservers'] %}
    server web{{ loop.index }} {{ hostvars[host].ansible_eth1.ipv4.address }}:{{ http_port }} check
    {% endfor %}

4. 默认变量文件（vars/default.yml） #

http_host: "hdvld"
http_conf: "hdvld.conf"
http_port: "80"
proxy_port: "8080"
disable_default: true
staticlb_port: "4331"

5. Vagrant自动生成的Inventory文件 #

#    Generated by Vagrant
HDVLD-TEST-LB01 ansible_host=127.0.0.1 ansible_port=2200 ansible_user='vagrant' ansible_ssh_private_key_file='/home/web01/VM2022/template/.vagrant/machines/HDVLD-TEST-LB01/virtualbox/private_key'
HDVLD-TEST-WEB02 ansible_host=127.0.0.1 ansible_port=2201 ansible_user='vagrant' ansible_ssh_private_key_file='/home/web01/VM2022/template/.vagrant/machines/HDVLD-TEST-WEB02/virtualbox/private_key'
HDVLD-TEST-WEB01 ansible_host=127.0.0.1 ansible_port=2222 ansible_user='vagrant' ansible_ssh_private_key_file='/home/web01/VM2022/template/.vagrant/machines/HDVLD-TEST-WEB01/virtualbox/private_key'
[webservers]
HDVLD-TEST-WEB01
HDVLD-TEST-WEB02
[loadbalancer]
HDVLD-TEST-LB01

故障根因与修复方案 #

故障1：无法读取ansible_eth1变量 #

• 根因1：Web节点Playbook中的facts采集任务错误添加delegate_facts: True参数，该参数会将采集到的主机事实委托给执行节点（即当前的LB节点）存储，不会写入对应Web节点的hostvars条目，LB节点渲染模板时自然读不到Web节点的网卡信息
• 根因2：Web节点Playbook中的UPurge任务写法错误，apt模块使用purge=yes参数时必须指定要卸载的包名，该任务执行失败会直接中断Web节点的Playbook流程，导致后续任务不执行、主机事实无法正常同步
• 根因3：Ubuntu Trusty 14.04环境下，Vagrant添加的私网网卡不一定固定命名为eth1，部分场景下会识别为eth2或其他命名，硬编码ansible_eth1会出现属性不存在的问题
• 修复方案：
  • 删除Web节点Playbook中Gather facts from new server、Debug facts from Server两个任务的delegate_facts: True配置，直接使用Playbook全局开启的gather_facts: True采集全量主机事实即可，不需要单独调用setup模块加过滤
  • 直接删除写法错误的UPurge任务，或补充要卸载的包名参数
  • 模板中不要硬编码网卡名，改为动态匹配私网网卡，将后端配置段替换为：

    backend app
        balance     roundrobin
        {% for host in groups['webservers'] %}
        {% set web_intf = hostvars[host].ansible_interfaces | select('match', 'eth[0-9]') | sort | last %}
        server web{{ loop.index }} {{ hostvars[host]['ansible_' + web_intf].ipv4.address }}:{{ http_port }} check
        {% endfor %}
    

故障2：8080端口连接被拒绝 #

• 根因1：LB节点任务顺序错误，在写入自定义HAProxy配置之前就执行了Starting haproxy任务，初始默认配置启动的HAProxy不会加载后续写入的自定义配置，即使配置变更触发了重启，若配置存在语法错误会直接导致HAProxy进程退出，端口无监听
• 根因2：HAProxy配置逻辑错误，listen haproxy-monitoring *:{{ proxy_port }}仅声明了监听8080端口，但没有配置统计页面的基础参数（如stats enable、访问权限规则），同时业务前端frontend main绑定的是80端口而非8080端口，8080端口本身没有配置正常的响应逻辑
• 根因3：防火墙规则配置顺序错误，在HAProxy配置写入、服务重启之后才添加8080端口的UFW允许规则，服务重启时防火墙还未放通对应端口，入站请求会被直接拒绝
• 修复方案：
  • 调整LB节点Playbook任务顺序：先安装HAProxy包，再写入配置文件，再配置防火墙放通规则，最后执行配置语法校验并启动/重启HAProxy服务，不要提前启动服务
  • 修正HAProxy监听配置：如果8080作为业务访问端口，将frontend main的bind配置改为bind *:{{ proxy_port }}；如果8080作为监控统计端口，补全监控配置：

    listen haproxy-monitoring *:{{ proxy_port }}
        stats enable
        stats uri /stats
        stats auth admin:admin123
    

  • 配置写入后先执行haproxy -c -f /etc/haproxy/haproxy.cfg校验配置语法，确认无报错后再重启服务
  • 服务启动后在LB节点执行ss -tulnp | grep haproxy确认对应端口正常处于LISTEN状态，再从客户端测试访问

内容的提问来源于stack exchange，提问作者Chris