使用Python Hypothesis测试API遇Flaky错误及重复规则名问题

我在测试通过API创建策略的场景，该API不允许重复的规则名称。使用Hypothesis做测试重试时，必然触发「不允许重复名称」的错误。于是添加「生成同名规则则先删除该策略」的逻辑后，遇到如下错误：

hypothesis.errors.Flaky: Inconsistent data generation! Data generation behaved differently between different runs. Is your data generation depending on external state?

相关代码 #

import string
from hypothesis import assume
import pytest
import unittest
from hypothesis import strategies as st, reject
from hypothesis.stateful import RuleBasedStateMachine, precondition, rule


used_names = set()


@st.composite
def unique_name_strategy(draw):
    name_strategy = st.text(min_size=5, max_size=31, alphabet=string.ascii_letters + string.digits)
    while True:
        name = draw(name_strategy)
        if name not in used_names:
            used_names.add(name)
            return name


class TestsEdlp(RuleBasedStateMachine):
    def __init__(self) -> None:
        super().__init__()

    rule_names = []
    tries = 0


    def rule_creation(self, rule_name):
        print("in rule creation")

        if rule_name in self.rule_names:
            self.rule_names.remove(rule_name)

        if rule_name:
            self.rule_names.append(rule_name)
            self.tries += 1
            return True
        else:
            return False


    @precondition(lambda self: self.tries < 10)
    @rule(rule_name=unique_name_strategy())
    def create_dlp_rule(self, rule_name):

        print(f"in function create rule {rule_name} tries: {self.tries}")
        is_success = self.rule_creation(rule_name=rule_name)
        assert is_success

TestsEdlp: unittest.TestCase = TestsEdlp.TestCase

问题根源 #

1. 全局变量导致状态污染：used_names是全局集合，测试运行后不会自动重置，不同测试用例、不同运行之间会共享这个状态，破坏Hypothesis数据生成的可复现性，触发Flaky错误。
2. 类属性误用：rule_names和tries是类属性，所有测试实例会共享这两个值，导致状态混乱。

修复方案 #

修正后的代码 #

import string
import pytest
import unittest
from hypothesis import strategies as st, reject
from hypothesis.stateful import RuleBasedStateMachine, precondition, rule


class TestsEdlp(RuleBasedStateMachine):
    def __init__(self) -> None:
        super().__init__()
        # 改为实例属性，每个测试实例独立维护状态
        self.rule_names = []
        self.tries = 0
        self.used_names = set()

    def rule_creation(self, rule_name):
        print("in rule creation")

        if rule_name in self.rule_names:
            self.rule_names.remove(rule_name)

        if rule_name:
            self.rule_names.append(rule_name)
            self.tries += 1
            return True
        else:
            return False

    @precondition(lambda self: self.tries < 10)
    @rule(rule_name=st.text(min_size=5, max_size=31, alphabet=string.ascii_letters + string.digits))
    def create_dlp_rule(self, rule_name):
        # 处理重复名称：要么跳过重新生成，要么执行删除逻辑
        if rule_name in self.used_names:
            # 若要Hypothesis重新生成名称，启用下面一行
            # reject()
            # 若要执行删除逻辑，调用已有方法
            self.rule_creation(rule_name)
        
        self.used_names.add(rule_name)
        print(f"in function create rule {rule_name} tries: {self.tries}")
        is_success = self.rule_creation(rule_name=rule_name)
        assert is_success


TestsEdlp: unittest.TestCase = TestsEdlp.TestCase

关键改动说明 #

• 移除全局状态：把used_names移到状态机实例内部，每个测试用例的状态完全独立。
• 类属性转实例属性：在__init__方法中初始化rule_names和tries，避免不同实例共享状态。
• 策略与状态解耦：不再用依赖全局状态的复合策略，直接在规则方法内处理重复名称，保证数据生成的可复现性。

内容的提问来源于stack exchange，提问作者Prabhleen Singh