Gemini API图像识别异常导致TurtleBot3红球检测任务失败求助

我现在在做一个TurtleBot3 waffle_pi的项目，想让它通过Gemini API识别红球然后把球撞开，但不管怎么折腾都没法正常工作。

一开始我以为是prompt写得有问题，因为API只返回过一个命令“A”，我反复修改prompt也没用。后来我干脆让API描述它看到的内容，结果离谱的是，明明画面里是平面上的红球，它却说这是一份医疗文书？

我现在搞不清到底是Gemini的图像识别能力有问题，还是我把图像转换成token传给API的方式不对，希望有人能帮我排查一下问题。

下面是我写的代码：

import time
import rclpy
from rclpy.node import Node
from sensor_msgs.msg import Image
from cv_bridge import CvBridge
from geometry_msgs.msg import Twist
import cv2
import google.generativeai as genai
import base64

class TurtleBot3CenterAndMove(Node):
    def __init__(self):
        super().__init__('turtlebot3_center_and_move')


        genai.configure(api_key="API_HERE")

        self.bridge = CvBridge()
        self.subscription = self.create_subscription(
            Image, '/camera/image_raw', self.image_callback, 10
        )
        self.cmd_vel_pub = self.create_publisher(Twist, '/cmd_vel', 10)

        self.get_logger().info('TurtleBot3 Center and Move Node Initialized.')

    def image_callback(self, msg):
        try:
            # Stop before processing the next command
            self.stop_robot()
            time.sleep(1)

            # Convert ROS Image to OpenCV format
            cv_image = self.bridge.imgmsg_to_cv2(msg, 'bgr8')
            cv2.imshow("Frame", cv_image)
            cv2.waitKey(2)

            # Encode the image to base64
            image_base64 = self.encode_image_to_base64(cv_image)

            # Create the prompt for the Gemini API
            prompt = self.create_prompt(image_base64)

            # Query Gemini API for the command
            gemini_command = self.get_gemini_command(prompt)

            if gemini_command:
                self.apply_teleop_command(gemini_command)
            else:
                self.get_logger().info("No valid command received. Stopping the robot.")
                self.apply_teleop_command('STOP')

        except Exception as e:
            self.get_logger().error(f"Error processing image: {e}")

    def encode_image_to_base64(self, cv_image):
        """
        Encodes the given OpenCV image to base64 for transmission.
        """
        _, encoded_image = cv2.imencode('.jpg', cv_image)
        byte_array = encoded_image.tobytes()
        return base64.b64encode(byte_array).decode('utf-8')

    def create_prompt(self, image_base64):
        """
        Creates the prompt for the Gemini API with the provided image base64.
        """
        prompt = (
            "whats in the image?"


            "Here is the image token for analysis:\n"
            f"{image_base64}\n\n"


        )
        return prompt

    def get_gemini_command(self, prompt):
        """
        Queries the Gemini API with the prompt and returns the generated command.
        """
        try:
            # Send the prompt to the Gemini API for processing
            model = genai.GenerativeModel(model_name="gemini-1.5-flash")
            response = model.generate_content(prompt)
            command = response.text.strip().upper()

            self.get_logger().info(f"Gemini Command: {command}")
            return command

        except Exception as e:
            self.get_logger().error(f"Error querying Gemini API: {e}")
            return None

    def apply_teleop_command(self, command):
        """
        Publishes a teleoperation command to control the robot's movements.
        """
        twist_msg = Twist()

        if command == 'A':  # Turn left
            twist_msg.angular.z = 0.2
        elif command == 'D':  # Turn right
            twist_msg.angular.z = -0.2
        elif command == 'W':  # Move forward
            twist_msg.linear.x = 0.2
        elif command == 'STOP':  # Stop
            twist_msg.linear.x = 0.0
            twist_msg.angular.z = 0.0

        self.cmd_vel_pub.publish(twist_msg)

        # Stop after executing the command
        if command != 'STOP':
            time.sleep(1)  # Delay before stopping
            self.stop_robot()

    def stop_robot(self):
        """
        Publishes a stop command to halt all robot movements.
        """
        twist_msg = Twist()
        twist_msg.linear.x = 0.0
        twist_msg.angular.z = 0.0
        self.cmd_vel_pub.publish(twist_msg)

def main(args=None):
    rclpy.init(args=args)
    node = TurtleBot3CenterAndMove()
    try:
        rclpy.spin(node)
    except KeyboardInterrupt:
        pass
    finally:
        node.destroy_node()
        rclpy.shutdown()

if __name__ == '__main__':
    main()

备注：内容来源于stack exchange，提问作者Rafael Topciov