如何在JavaScript中从WAV文件获取频谱图适配TensorFlowJS识别器？

Your current code is passing raw audio samples directly to recognizer.recognize(), but the model expects a log-mel spectrogram (not raw waveform data) as input. That's why you're getting inaccurate results. Let's walk through the correct process to convert your WAV file into the proper format.

Key Background #

The model's input shape (1, 43, 232) indicates it expects a batch of 1 spectrogram with:

• 43 time frames
• 232 mel frequency bins

Most speech recognition models (including the Speech Commands model) are trained on 16kHz mono audio, so we'll start by ensuring your WAV file meets these specs.

Step-by-Step Solution #

1. Install Required Dependencies #

Make sure you have these packages installed:

npm install @tensorflow/tfjs-node @tensorflow-models/speech-commands wav

2. Complete Code Implementation #

This code reads the WAV file, processes it into a log-mel spectrogram, and feeds it to the recognizer:

const fs = require('fs');
const wav = require('wav');
const tf = require('@tensorflow/tfjs-node');
const speechCommands = require('@tensorflow-models/speech-commands');

async function recognizeWavFile(filePath) {
    // Load the pre-trained speech recognizer
    const recognizer = speechCommands.create('BROWSER_FFT');
    await recognizer.ensureModelLoaded();
    const inputShape = recognizer.modelInputShape();
    const numFrames = inputShape[1];
    const numMelBins = inputShape[2];
    const targetSampleRate = 16000; // Standard for speech models

    // Read and decode the WAV file
    const buffer = fs.readFileSync(filePath);
    const reader = new wav.Reader();
    let audioData = null;
    let sampleRate = null;

    // Handle WAV format metadata
    reader.on('format', (format) => {
        sampleRate = format.sampleRate;
        console.log(`WAV File Specs: ${sampleRate} Hz, ${format.channels} channels`);
    });

    // Convert raw WAV data to normalized Float32Array ([-1, 1])
    reader.on('data', (data) => {
        audioData = new Float32Array(data.length / 2);
        for (let i = 0; i < data.length; i += 2) {
            // Convert 16-bit PCM to normalized float
            audioData[i / 2] = data.readInt16LE(i) / 32768;
        }
    });

    // Wait for WAV parsing to complete
    reader.end(buffer);
    await new Promise(resolve => reader.on('end', resolve));

    if (!audioData) {
        throw new Error('Failed to read audio data from WAV file');
    }

    // Resample audio to target sample rate if needed
    if (sampleRate !== targetSampleRate) {
        const audioTensor = tf.tensor1d(audioData);
        const resampledTensor = await tf.audio.resample(audioTensor, sampleRate, targetSampleRate);
        audioData = resampledTensor.array();
        // Clean up tensors to avoid memory leaks
        audioTensor.dispose();
        resampledTensor.dispose();
    }

    // Convert audio to log-mel spectrogram
    const audioTensor = tf.tensor1d(audioData);
    const frameSize = 480; // 30ms window at 16kHz
    const frameStep = 160; // 10ms step at 16kHz

    // Compute Short-Time Fourier Transform (STFT)
    const stft = tf.signal.stft(audioTensor, frameSize, frameStep, { fftLength: 512 });
    // Get magnitude of complex STFT values
    const magnitude = tf.abs(stft);
    // Create mel filter banks matching model's mel bin count
    const melFilterBank = tf.signal.melFilterbanks(
        numMelBins,
        tf.signal.fftLengthForFrameSize(frameSize),
        targetSampleRate,
        20, // Lower frequency limit
        4000 // Upper frequency limit
    );
    // Apply mel filter to magnitude spectrogram
    const melSpectrogram = tf.matMul(magnitude, melFilterBank);
    // Convert to log scale (add epsilon to avoid log(0))
    const logMelSpectrogram = tf.log(melSpectrogram.add(1e-6));

    // Adjust spectrogram to match model's input frame count
    let inputSpectrogram = logMelSpectrogram;
    const currentFrames = inputSpectrogram.shape[0];
    
    if (currentFrames > numFrames) {
        // Slice to keep only the first N frames
        inputSpectrogram = inputSpectrogram.slice([0], [numFrames]);
    } else if (currentFrames < numFrames) {
        // Pad with zeros to reach required frame count
        const padAmount = numFrames - currentFrames;
        inputSpectrogram = tf.pad(inputSpectrogram, [[0, padAmount], [0, 0]]);
    }

    // Add batch dimension to match input shape (1, numFrames, numMelBins)
    const inputTensor = inputSpectrogram.expandDims(0);

    // Run recognition
    const result = await recognizer.recognize(inputTensor);
    console.log('Recognition Result:', result);

    // Clean up all tensors to prevent memory leaks
    audioTensor.dispose();
    stft.dispose();
    magnitude.dispose();
    melFilterBank.dispose();
    melSpectrogram.dispose();
    logMelSpectrogram.dispose();
    inputSpectrogram.dispose();
    inputTensor.dispose();

    return result;
}

// Run the function with your WAV file
recognizeWavFile('zero1.wav').catch(err => console.error('Error:', err));

Important Notes #

• Sample Rate Check: Ensure your WAV file is 16kHz. If not, the code automatically resamples it.
• Frame Parameters: The frameSize and frameStep values (30ms window, 10ms step) match the default parameters used by the Speech Commands model. If your custom model uses different values, adjust these accordingly.
• Memory Management: Always dispose of TensorFlow tensors after use to avoid memory leaks, especially in Node.js environments.
• Audio Length: Your audio clip should be at least ~430ms long (43 frames × 10ms step) to fill the model's input. If shorter, the code pads with zeros.

内容的提问来源于stack exchange，提问作者ARPIT PRASHANT BAHETY