如何在Android ARCore中用OpenGL（无Sceneform）实现触摸添加文本标签？

Got it, let's walk through how to replicate your Sceneform-based text label feature using pure OpenGL for your ARCore project. The core idea is to render your Android TextView to an OpenGL texture, map that texture onto a simple 3D plane mesh, and place it in AR space using ARCore's anchors—mirroring what Sceneform does under the hood, but handling the low-level bits ourselves.

Step 1: Render Your TextView to an OpenGL Texture #

First, we'll convert your custom TextView layout into a texture that OpenGL can use, replacing Sceneform's ViewRenderable functionality.

private int createTextTexture(String text) {
    // Recreate the same layout from your Sceneform code
    LinearLayout linearLayout = (LinearLayout) LayoutInflater.from(this).inflate(R.layout.text_view, null);
    
    // Configure the circular badge
    TextView dynamicTextBackground = new TextView(this);
    LinearLayout.LayoutParams badgeParams = new LinearLayout.LayoutParams(10, 10);
    badgeParams.setMargins(45, 0, 5, 0);
    dynamicTextBackground.setLayoutParams(badgeParams);
    dynamicTextBackground.setBackgroundDrawable(ContextCompat.getDrawable(this, R.drawable.badge_circle));
    dynamicTextBackground.setText("1");
    dynamicTextBackground.setTextSize(1);
    
    // Configure the main text view
    TextView dynamicTextView = new TextView(this);
    LinearLayout.LayoutParams textParams = new LinearLayout.LayoutParams(
        LinearLayout.LayoutParams.WRAP_CONTENT, 
        LinearLayout.LayoutParams.WRAP_CONTENT
    );
    dynamicTextView.setLayoutParams(textParams);
    dynamicTextView.setText(text);
    dynamicTextView.setBackgroundColor(Color.WHITE);
    dynamicTextView.setTextColor(Color.RED);
    dynamicTextView.setPadding(2, 2, 2, 2);
    
    linearLayout.addView(dynamicTextBackground);
    linearLayout.addView(dynamicTextView);
    
    // Measure and layout the view to get its actual size
    int widthSpec = View.MeasureSpec.makeMeasureSpec(0, View.MeasureSpec.UNSPECIFIED);
    int heightSpec = View.MeasureSpec.makeMeasureSpec(0, View.MeasureSpec.UNSPECIFIED);
    linearLayout.measure(widthSpec, heightSpec);
    linearLayout.layout(0, 0, linearLayout.getMeasuredWidth(), linearLayout.getMeasuredHeight());
    
    // Render the view to a Bitmap
    Bitmap bitmap = Bitmap.createBitmap(
        linearLayout.getMeasuredWidth(), 
        linearLayout.getMeasuredHeight(), 
        Bitmap.Config.ARGB_8888
    );
    Canvas canvas = new Canvas(bitmap);
    linearLayout.draw(canvas);
    
    // Convert Bitmap to OpenGL texture
    int[] textureId = new int[1];
    GLES20.glGenTextures(1, textureId, 0);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId[0]);
    
    // Set texture parameters for smooth rendering
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
    
    // Upload the bitmap to the OpenGL texture
    GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0);
    bitmap.recycle(); // Clean up unused bitmap to save memory
    
    return textureId[0];
}

Step 2: Create a Simple Plane Mesh for the Text #

We need a 2D quad (plane) to map our text texture onto. Define the vertex, UV coordinate, and index data:

// Vertex data: x, y, z (position), u, v (texture coordinates)
private float[] textPlaneVertices = {
    -0.5f,  0.5f, 0.0f, 0.0f, 1.0f, // Top-left
     0.5f,  0.5f, 0.0f, 1.0f, 1.0f, // Top-right
     0.5f, -0.5f, 0.0f, 1.0f, 0.0f, // Bottom-right
    -0.5f, -0.5f, 0.0f, 0.0f, 0.0f  // Bottom-left
};

// Indices to form two triangles (makes up the quad)
private short[] textPlaneIndices = {0, 1, 2, 0, 2, 3};

Step 3: Write OpenGL Shaders for Text Rendering #

We'll need vertex and fragment shaders to handle AR pose matrices and texture mapping:

Vertex Shader (res/raw/text_vertex_shader.glsl) #

uniform mat4 mvpMatrix;
attribute vec3 position;
attribute vec2 texCoord;
varying vec2 vTexCoord;

void main() {
    gl_Position = mvpMatrix * vec4(position, 1.0);
    vTexCoord = texCoord;
}

Fragment Shader (res/raw/text_fragment_shader.glsl) #

precision mediump float;
uniform sampler2D textTexture;
varying vec2 vTexCoord;

void main() {
    gl_FragColor = texture2D(textTexture, vTexCoord);
    // Discard fully transparent pixels to avoid unnecessary overdraw
    if (gl_FragColor.a < 0.1) discard;
}

Step 4: Handle AR Touch & Anchor Creation #

In your GLSurfaceView touch listener, perform an ARCore hit test to find a plane, create an anchor, and store the anchor along with its text texture:

// Helper class to store text label data
private static class TextLabel {
    Anchor anchor;
    int textureId;
    float scale; // Adjust to make text visible in AR space

    TextLabel(Anchor anchor, int textureId, float scale) {
        this.anchor = anchor;
        this.textureId = textureId;
        this.scale = scale;
    }
}

private List<TextLabel> textLabels = new ArrayList<>();

// Touch handler in your activity/GLSurfaceView
@Override
public boolean onTouch(View v, MotionEvent event) {
    if (event.getAction() == MotionEvent.ACTION_UP) {
        Frame frame = arSession.update();
        // Find the first valid plane hit
        TrackableHit hit = frame.hitTest(event).stream()
            .filter(h -> h.isHitInTrackable() && h.getTrackable() instanceof Plane)
            .findFirst()
            .orElse(null);
        
        if (hit != null) {
            Anchor anchor = hit.createAnchor();
            String customText = "Your Label Text"; // Replace with dynamic input if needed
            int textTexture = createTextTexture(customText);
            textLabels.add(new TextLabel(anchor, textTexture, 0.1f)); // Scale to fit AR space
        }
    }
    return true;
}

Step 5: Render Text Labels in the OpenGL Loop #

In your GLSurfaceView.Renderer's onDrawFrame method, iterate over all text labels, calculate their MVP matrix using the anchor's pose, and draw the textured plane:

private int textProgram;
private int mvpMatrixHandle;
private int positionHandle;
private int texCoordHandle;
private int textureHandle;
private float[] viewMatrix = new float[16];
private float[] projectionMatrix = new float[16];

// Initialize shader program in onSurfaceCreated
private void initTextShader() {
    String vertexShader = loadShaderRawResource(R.raw.text_vertex_shader);
    String fragmentShader = loadShaderRawResource(R.raw.text_fragment_shader);
    textProgram = createShaderProgram(vertexShader, fragmentShader);
    
    // Get shader uniform/attribute handles
    mvpMatrixHandle = GLES20.glGetUniformLocation(textProgram, "mvpMatrix");
    positionHandle = GLES20.glGetAttribLocation(textProgram, "position");
    texCoordHandle = GLES20.glGetAttribLocation(textProgram, "texCoord");
    textureHandle = GLES20.glGetUniformLocation(textProgram, "textTexture");
}

// In onDrawFrame:
@Override
public void onDrawFrame(GL10 gl) {
    // ... Your existing ARCore frame rendering code ...
    // Make sure to update viewMatrix and projectionMatrix from ARCore's frame
    
    // Enable transparency for text backgrounds
    GLES20.glEnable(GLES20.GL_BLEND);
    GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
    
    GLES20.glUseProgram(textProgram);
    
    // Bind vertex data (use VBOs for better performance in production)
    GLES20.glVertexAttribPointer(positionHandle, 3, GLES20.GL_FLOAT, false, 5 * Float.SIZE, ByteBuffer.wrap(textPlaneVertices));
    GLES20.glEnableVertexAttribArray(positionHandle);
    
    GLES20.glVertexAttribPointer(texCoordHandle, 2, GLES20.GL_FLOAT, false, 5 * Float.SIZE, ByteBuffer.wrap(textPlaneVertices).position(3));
    GLES20.glEnableVertexAttribArray(texCoordHandle);
    
    // Draw each text label
    for (TextLabel label : textLabels) {
        // Skip labels with untracked anchors
        if (!label.anchor.getTrackingState().equals(TrackingState.TRACKING)) continue;
        
        // Calculate model matrix from anchor pose
        float[] modelMatrix = new float[16];
        label.anchor.getPose().toMatrix(modelMatrix, 0);
        
        // Apply scale to adjust text size in AR space
        Matrix.scaleM(modelMatrix, 0, label.scale, label.scale, label.scale);
        
        // Combine model, view, and projection matrices
        float[] mvpMatrix = new float[16];
        Matrix.multiplyMM(mvpMatrix, 0, viewMatrix, 0, modelMatrix, 0);
        Matrix.multiplyMM(mvpMatrix, 0, projectionMatrix, 0, mvpMatrix, 0);
        
        // Set shader uniforms
        GLES20.glUniformMatrix4fv(mvpMatrixHandle, 1, false, mvpMatrix, 0);
        GLES20.glUniform1i(textureHandle, 0);
        
        // Bind texture and draw the quad
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, label.textureId);
        GLES20.glDrawElements(GLES20.GL_TRIANGLES, textPlaneIndices.length, GLES20.GL_UNSIGNED_SHORT, ByteBuffer.wrap(textPlaneIndices));
    }
    
    // Clean up
    GLES20.glDisable(GLES20.GL_BLEND);
    GLES20.glUseProgram(0);
}

// Helper methods for shader loading/compilation
private String loadShaderRawResource(int resourceId) {
    InputStream is = getResources().openRawResource(resourceId);
    BufferedReader reader = new BufferedReader(new InputStreamReader(is));
    StringBuilder sb = new StringBuilder();
    String line;
    try {
        while ((line = reader.readLine()) != null) {
            sb.append(line).append("\n");
        }
        reader.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return sb.toString();
}

private int createShaderProgram(String vertexShader, String fragmentShader) {
    int vertexShaderId = compileShader(GLES20.GL_VERTEX_SHADER, vertexShader);
    int fragmentShaderId = compileShader(GLES20.GL_FRAGMENT_SHADER, fragmentShader);
    
    int programId = GLES20.glCreateProgram();
    GLES20.glAttachShader(programId, vertexShaderId);
    GLES20.glAttachShader(programId, fragmentShaderId);
    GLES20.glLinkProgram(programId);
    
    // Check for link errors
    int[] linkStatus = new int[1];
    GLES20.glGetProgramiv(programId, GLES20.GL_LINK_STATUS, linkStatus, 0);
    if (linkStatus[0] == 0) {
        Log.e("OpenGL", "Shader program link failed: " + GLES20.glGetProgramInfoLog(programId));
        GLES20.glDeleteProgram(programId);
        return 0;
    }
    
    return programId;
}

private int compileShader(int type, String shaderCode) {
    int shaderId = GLES20.glCreateShader(type);
    GLES20.glShaderSource(shaderId, shaderCode);
    GLES20.glCompileShader(shaderId);
    
    // Check for compile errors
    int[] compileStatus = new int[1];
    GLES20.glGetShaderiv(shaderId, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
    if (compileStatus[0] == 0) {
        Log.e("OpenGL", "Shader compile failed: " + GLES20.glGetShaderInfoLog(shaderId));
        GLES20.glDeleteShader(shaderId);
        return 0;
    }
    
    return shaderId;
}

Key Notes & Optimizations #

• Resource Cleanup: Delete textures and anchors when they're no longer needed (e.g., when the anchor is destroyed) to avoid memory leaks.
• VBOs: For better performance, use Vertex Buffer Objects (VBOs) instead of passing vertex data directly each frame.
• Text Scaling: Adjust the scale value in TextLabel to make text visible at typical AR distances—tweak it based on your text size.
• Tracking State: Only draw labels when their anchor is in TRACKING state to avoid jitter or invisible elements.

内容的提问来源于stack exchange，提问作者Nitin Thakor