Ansible异步监控服务器状态并触发后续任务的技术问询

需要用Ansible实现：主动监控多台服务器状态，当任意一台服务器状态变为“OK”时，立即为该服务器执行后续任务，同时继续监控其余未达标服务器。

当前使用Ansible版本：

ansible [core 2.14.3]
  config file = /home/user/.ansible.cfg
  configured module search path = ['/home/user/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
  ansible python module location = /usr/lib/python3/dist-packages/ansible
  ansible collection location = /home/user/.ansible/collections:/usr/share/ansible/collections
  executable location = /usr/bin/ansible
  python version = 3.11.2 (main, Mar 13 2023, 12:18:29) [GCC 12.2.0] (/usr/bin/python3)
  jinja version = 3.1.2
  libyaml = True

现有通过自定义state Role读取/tmp状态文件的监控逻辑，但当前同步循环最多执行5次，无法满足“达标即触发任务+持续监控”的需求，寻求可行方案。

现有配置 #

目录结构 #

"{{playbook_dir}}/testing/main.yaml"
"{{playbook_dir}}/roles/state/tasks/get.yaml"
/tmp/localhost.state
/tmp/127.0.0.1.state
/tmp/127.0.1.1.state

main.yaml内容 #

---
- hosts: [localhost]
  gather_facts: false
  vars:
    servers_state_files: {
      "localhost": "/tmp/localhost.state",
      "127.0.0.1": "/tmp/127.0.0.1.state",
      "127.0.1.1": "/tmp/127.0.1.1.state",
    }

  tasks:
    - name: "task 1 - get and monitor server/s state"
      include_role:
        name: state
        tasks_from: get.yaml
      vars:
        - server_names: "{{ servers_state_files.keys() }}"
      # async: 100
      # poll: 0
      register: get_server_state_task
    
    - name: "task 2 - do some actions on server/s that have an 'OK' state"
      debug:
        msg: "some action on server - {{ item }}"
      loop: "{{ server_state_dict_ok.keys() }}"
      retries: 5
      delay: 10
      until: server_state_dict_ok[item].state == 'OK'
      when: server_state_dict_ok is defined

get.yaml内容 #

---
- name: "open state file/s"
  debug:
    msg:
      - "server state file - {{ servers_state_files[item] }}"
      - "state - {{ lookup('file', servers_state_files[item]) }}"
  loop: "{{ server_names }}"
  when:
    - server_names is defined
    - servers_state_files is defined

- name: "set server state dictionary fact"
  set_fact:
    server_state_dict: "{{ server_state_dict | default({}) | combine( {item: {'state': lookup('file', servers_state_files[item])}} ) }}"
  loop: "{{ server_names }}"
  when: 
    - server_names is defined
    - servers_state_files is defined

- name: "show server_state_dict param"
  debug:
    msg: 
      - "server_state_dict - {{ server_state_dict }}"
  when: server_state_dict is defined

- name: "create server_state_dict_ok"
  set_fact:
    server_state_dict_ok: "{{ server_state_dict_ok | default({}) | combine( {item: {'state': lookup('file', servers_state_files[item])}} ) }}"
  with_items: "{{ server_state_dict.keys() }}"
  when: 
    - server_state_dict is defined
    - server_state_dict[item].state == 'OK'

- name: "Show server_state_dict_ok param"
  debug:
    msg: 
      - "server_state_dict_ok - {{ server_state_dict_ok }}"
  when: 
    - server_state_dict_ok is defined
    - server_state_dict_ok | length > 0

- name: "'Pop' keys from server_state_dict that have 'OK' state"
  set_fact:
    server_state_dict: "{{ server_state_dict | ansible.utils.remove_keys(target=[item]) }}"
  with_items: "{{ server_state_dict }}"
  when: 
    - server_state_dict is defined
    - server_state_dict[item].state == 'OK'

- name: "Show server_state_dict param"
  debug:
    msg: 
      - "server_state_dict - {{ server_state_dict }}"
  when: 
    - server_state_dict is defined
    - server_state_dict | length > 0

- name: "Set and increment counter"
  set_fact:
    counter: "{{ counter | default(0) | int + 1 | int }}"
  when: 
    - server_state_dict is defined
    - server_state_dict | length > 0

- name: "Show counter param"
  debug:
    msg: 
      - "counter - {{ counter | int }}"
  when: 
    - counter is defined

- name: "Set a 'sleep' of 10 seconds to give the server/s a chance to have it's state changed to 'OK'"
  command: "sleep 10"
  when: 
    - server_state_dict is defined
    - server_state_dict | length > 0

- name: "Continue to monitor server/s state for servers whose state is not 'OK'"
  include_tasks: get.yaml
  vars:
    server_names: "{{ server_state_dict.keys() }}"
  when: 
    - server_state_dict is defined
    - server_state_dict | length > 0
    - counter | int <= 5

可行解决方案 #

方案1：异步监控+轮询触发任务 #

将监控逻辑拆分为后台异步任务，主流程轮询检查达标服务器，触发任务后移除已达标节点，继续监控剩余服务器。

修改main.yaml：

---
- hosts: localhost
  gather_facts: false
  vars:
    servers_state_files: {
      "localhost": "/tmp/localhost.state",
      "127.0.0.1": "/tmp/127.0.0.1.state",
      "127.0.1.1": "/tmp/127.0.1.1.state",
    }
    remaining_servers: "{{ servers_state_files.keys() | list }}"
    max_retries: 20
    poll_interval: 10

  tasks:
    - name: 启动后台监控任务
      include_role:
        name: state
        tasks_from: monitor_single_batch.yaml
      vars:
        target_servers: "{{ remaining_servers }}"
      async: "{{ max_retries * poll_interval }}"
      poll: 0
      register: monitor_job

    - name: 轮询检查达标服务器并执行任务
      block:
        - name: 获取监控任务结果
          async_status:
            jid: "{{ monitor_job.ansible_job_id }}"
          register: job_result
          until: job_result.finished or job_result.result.new_ok_servers | length > 0
          retries: "{{ max_retries }}"
          delay: "{{ poll_interval }}"

        - name: 对新达标服务器执行任务
          debug:
            msg: "执行任务 - {{ item }}"
          loop: "{{ job_result.result.new_ok_servers }}"

        - name: 更新剩余监控服务器列表
          set_fact:
            remaining_servers: "{{ remaining_servers | difference(job_result.result.new_ok_servers) }}"

        - name: 继续监控剩余服务器
          include_tasks: "{{ playbook_dir }}/roles/state/tasks/continue_monitoring.yaml"
          when: remaining_servers | length > 0
      until: remaining_servers | length == 0
      retries: "{{ max_retries }}"

创建roles/state/tasks/monitor_single_batch.yaml（单次检查状态并返回达标列表）：

---
- name: 检查目标服务器状态
  set_fact:
    current_ok_servers: "{{ current_ok_servers | default([]) + [item] }}"
  loop: "{{ target_servers }}"
  when: lookup('file', servers_state_files[item]) == 'OK'

- name: 返回新达标服务器
  set_fact:
    new_ok_servers: "{{ current_ok_servers | default([]) }}"

方案2：单服务器独立异步监控 #

为每个服务器单独设置异步监控循环，达标后立即触发任务，实现并行监控逻辑：

修改main.yaml：

---
- hosts: localhost
  gather_facts: false
  vars:
    servers_state_files: {
      "localhost": "/tmp/localhost.state",
      "127.0.0.1": "/tmp/127.0.0.1.state",
      "127.0.1.1": "/tmp/127.0.1.1.state",
    }
    max_retries: 20
    poll_interval: 10

  tasks:
    - name: 监控单个服务器状态并执行任务
      block:
        - name: 等待服务器状态变为OK
          wait_for:
            path: "{{ servers_state_files[item] }}"
            search_regex: "^OK$"
            timeout: "{{ max_retries * poll_interval }}"
            delay: "{{ poll_interval }}"

        - name: 对达标服务器执行任务
          debug:
            msg: "执行任务 - {{ item }}"
      loop: "{{ servers_state_files.keys() }}"
      async: "{{ max_retries * poll_interval }}"
      poll: 0

方案3：Python脚本辅助监控 #

如果Ansible原生逻辑不够灵活，可编写Python脚本实现后台监控，达标后调用Ansible任务：

脚本monitor_states.py：

import time
import subprocess
import os

STATE_FILES = {
    "localhost": "/tmp/localhost.state",
    "127.0.0.1": "/tmp/127.0.0.1.state",
    "127.0.1.1": "/tmp/127.0.1.1.state"
}
MAX_RETRIES = 20
POLL_INTERVAL = 10

def check_state(server, path):
    if not os.path.exists(path):
        return False
    with open(path, 'r') as f:
        return f.read().strip() == 'OK'

def run_ansible_task(server):
    subprocess.run([
        "ansible-playbook", "-i", "localhost,", "-e", f"target_server={server}", 
        "/path/to/your/task_playbook.yaml"
    ])

def main():
    pending_servers = list(STATE_FILES.keys())
    retries = 0
    while pending_servers and retries < MAX_RETRIES:
        for server in pending_servers.copy():
            if check_state(server, STATE_FILES[server]):
                run_ansible_task(server)
                pending_servers.remove(server)
        retries += 1
        time.sleep(POLL_INTERVAL)

if __name__ == "__main__":
    main()

对应的任务剧本task_playbook.yaml：

---
- hosts: localhost
  gather_facts: false
  vars:
    target_server: ""
  tasks:
    - name: 执行目标服务器任务
      debug:
        msg: "执行任务 - {{ target_server }}"

内容的提问来源于stack exchange，提问作者FireHawk2300