Unity ML Agents行为克隆场景下OnActionReceived未调用问题

问题现象 #

• 基于纯行为克隆（Behavioural Cloning）开发泊车Agent，未使用DecisionRequester、奖励逻辑等组件
• 录制演示后，演示文件仅记录1步和2个episode，但实际录制了更多内容
• Debug日志显示碰撞障碍物或泊车后会启动新episode，但记录的episode数仍为2
• OnActionReceived()方法中的Debug.Log从未输出，说明该方法从未被调用

核心代码 #

using System.Collections;
using System.Collections.Generic;
using Unity.MLAgents;
using Unity.MLAgents.Sensors;
using UnityEngine;
using Unity.MLAgents.Actuators;

public class CarParkingAgent : Agent
{
    public Transform targetParkingSpot;
    public CarController carController;
    public float maxRaycastDistance = 10f;
    private Rigidbody rb;
    public float maxSpeed = 10f;
    public float maxAngularVelocity = 10f;
    private bool isCarParked = false;
    public Vector3 tensor;
    private bool lastFrameParkedStatus = false;
    private int currentStep = 0;
    private bool useHeuristic = true; // after ur done with BC just delete this whole logic

    void Start()
    {
        rb = GetComponent<Rigidbody>();
        rb.inertiaTensor = tensor;
        rb.inertiaTensor = new Vector3(1829.532f, 1974.514f, 391.8728f);

        Debug.Log("Initial Car Position: " + transform.localPosition);
    }

    private void Update()
    {
        // Check if the car is parked
        if (!isCarParked)
        {
            Debug.Log("car aint parked");
            isCarParked = IsCarOnParkingSpot();
        }
    }
    private void FixedUpdate()
    {
        //idk what this does but if i remove it my entire IsCarOnParkingSpot() stops working
        if (!lastFrameParkedStatus)
        {
            isCarParked = IsCarOnParkingSpot();
        }
        lastFrameParkedStatus = isCarParked;
    }

    private void Heuristic()
    {
        if (!useHeuristic)
            return;

        // Calculate target direction (normalized)
        Vector3 toTarget = targetParkingSpot.localPosition - transform.localPosition;
        Vector3 normalizedToTarget = toTarget.normalized;

        // Apply heuristic actions based on target direction
        float steering = Vector3.Dot(normalizedToTarget, transform.right);
        float accelerate = Mathf.Clamp01(Vector3.Dot(rb.velocity, transform.forward));
        float brake = 0f;

        // Apply actions
        carController.HandleMotor(accelerate, brake);
        carController.HandleSteering(steering);
    }

    public override void CollectObservations(VectorSensor sensor)
    {
        
        sensor.AddObservation(transform.localPosition);
        sensor.AddObservation(transform.rotation);
        sensor.AddObservation(carController.GetVelocity());

        Vector3 toTarget = targetParkingSpot.localPosition - transform.localPosition;
        Vector3 normalizedToTarget = toTarget.normalized;
        sensor.AddObservation(toTarget.magnitude);
        sensor.AddObservation(normalizedToTarget);
        sensor.AddObservation(rb.velocity.magnitude / maxSpeed);
        sensor.AddObservation(rb.angularVelocity.magnitude / maxAngularVelocity);

        sensor.AddObservation(targetParkingSpot.localPosition);
        sensor.AddObservation(targetParkingSpot.rotation);

        Vector3[] raycastDirections = { transform.forward, transform.right, -transform.right, transform.forward + transform.right, transform.forward - transform.right };

        foreach (Vector3 direction in raycastDirections)
        {
            if (Physics.Raycast(transform.localPosition, direction, out RaycastHit hit, maxRaycastDistance))
            {
                float normalizedDistance = hit.distance / maxRaycastDistance;
                sensor.AddObservation(normalizedDistance);
            }
            else
            {
                sensor.AddObservation(-1f);
            }
        }
    }

    public override void OnActionReceived(ActionBuffers actions)
    {
        Debug.Log("OnActionReceived called, current step = " + currentStep);
        currentStep++;
        float accelerate = actions.ContinuousActions[0];
        float brake = actions.ContinuousActions[1];
        float steering = actions.ContinuousActions[2];

        if (useHeuristic)
        {
            Heuristic();
        }
        else
        {
            // Use the agent's policy to handle actions
            carController.HandleMotor(accelerate, brake);
            carController.HandleSteering(steering);
        }
    }

    public override void OnEpisodeBegin()
    {
        currentStep = 0;
        Debug.Log("new episode begun");
        ResetCarPosition();
        isCarParked = false;
        lastFrameParkedStatus = false;
    }

    public void OnTriggerEnter(Collider collision)
    {
        if (collision.gameObject.CompareTag("Obstacle"))
        {
            ResetCarPosition();
            EndEpisode();
            isCarParked = false;
        }
        else if (collision.gameObject.CompareTag("ParkingSpot") && !isCarParked)
        {
            IsCarOnParkingSpot();
        }
    }

    public bool IsCarOnParkingSpot()
    {
        Vector3 toParkingSpot = targetParkingSpot.position - transform.position;
        //Debug.Log("Distance from parking spot: " + toParkingSpot.magnitude);
        if (toParkingSpot.magnitude < 1.5f) 
        { 
            isCarParked = true;
            ResetCarPosition();
            EndEpisode();
            return true;
        }
        isCarParked = false;

        return false;
    }

    private void ResetCarPosition()
    {
        rb.velocity = Vector3.zero;
        rb.angularVelocity = Vector3.zero;

        Vector3 localPosition = new Vector3(
            UnityEngine.Random.Range(-7f, -4f),
            0.1f,
            UnityEngine.Random.Range(-7f, 2f));

        //Debug.Log("Calculated Local Position: " + localPosition);

        transform.localPosition = localPosition;
        transform.localRotation = Quaternion.Euler(Vector3.zero);
        
    }
}

原因分析 #

1. 缺少决策触发机制：即使是行为克隆模式，ML-Agents仍需要触发决策请求才能调用OnActionReceived()。未添加DecisionRequester组件，也未手动调用RequestDecision()，导致Agent从未收到决策信号，方法自然不会执行。

2. Heuristic逻辑执行方式错误：当前Heuristic()是私有方法，直接在Update/FixedUpdate中执行，绕过了ML-Agents的行为记录流程，导致演示数据无法捕获动作，同时也让OnActionReceived()失去了触发意义。

3. Episode结束逻辑顺序错误：IsCarOnParkingSpot()和OnTriggerEnter()中先重置位置再调用EndEpisode()，打乱了ML-Agents的episode生命周期流程，导致步骤记录不完整。

修复方案 #

1. 添加决策触发逻辑 #

在FixedUpdate()中手动调用RequestDecision()，确保Agent持续接收决策信号：

private void FixedUpdate()
{
    RequestDecision(); // 关键：触发决策请求，让OnActionReceived被调用
    if (!lastFrameParkedStatus)
    {
        isCarParked = IsCarOnParkingSpot();
    }
    lastFrameParkedStatus = isCarParked;
}

2. 修正Heuristic实现方式 #

重写ML-Agents标准的Heuristic()方法，将动作写入动作缓冲区，让OnActionReceived()统一处理：

public override void Heuristic(in ActionBuffers actionsOut)
{
    if (!useHeuristic) return;

    Vector3 toTarget = targetParkingSpot.localPosition - transform.localPosition;
    Vector3 normalizedToTarget = toTarget.normalized;

    float steering = Vector3.Dot(normalizedToTarget, transform.right);
    float accelerate = Mathf.Clamp01(Vector3.Dot(rb.velocity, transform.forward));
    float brake = 0f;

    // 将动作写入缓冲区，供OnActionReceived使用
    actionsOut.ContinuousActions[0] = accelerate;
    actionsOut.ContinuousActions[1] = brake;
    actionsOut.ContinuousActions[2] = steering;
}

同时修改OnActionReceived()，移除独立的Heuristic调用：

public override void OnActionReceived(ActionBuffers actions)
{
    Debug.Log("OnActionReceived called, current step = " + currentStep);
    currentStep++;
    float accelerate = actions.ContinuousActions[0];
    float brake = actions.ContinuousActions[1];
    float steering = actions.ContinuousActions[2];

    // 统一应用动作，无论是否使用启发式
    carController.HandleMotor(accelerate, brake);
    carController.HandleSteering(steering);
}

3. 调整Episode结束逻辑顺序 #

先结束Episode再重置位置，确保流程符合ML-Agents规范：

public bool IsCarOnParkingSpot()
{
    Vector3 toParkingSpot = targetParkingSpot.position - transform.position;
    if (toParkingSpot.magnitude < 1.5f) 
    { 
        isCarParked = true;
        EndEpisode(); // 先结束Episode，再重置位置
        ResetCarPosition();
        return true;
    }
    isCarParked = false;
    return false;
}

public void OnTriggerEnter(Collider collision)
{
    if (collision.gameObject.CompareTag("Obstacle"))
    {
        EndEpisode();
        ResetCarPosition();
        isCarParked = false;
    }
    else if (collision.gameObject.CompareTag("ParkingSpot") && !isCarParked)
    {
        IsCarOnParkingSpot();
    }
}

4. 检查Agent配置 #

• 在Inspector面板中，将Agent的Behavior Type设置为Heuristic Only（录制演示时）或Default（训练时）
• 确认Action Spec中连续动作数量设置为3（对应accelerate、brake、steering）

内容的提问来源于stack exchange，提问作者smonk