接口说明-WebSocket

接入必读

请先查看接入必读了解具体接入方式，再参考此文档完成接入。

功能介绍

VoiceConversionStream为用户提供声音转换能力，支持用户输入人声音频，并通过深度学习转化为其他指定的音色，高度保留输入语音的说话风格、情感变化、说话节奏。流式声音转换支持实时的转换效果。

• 输入：原始说话语音二进制数据
• 输出：转换音色后的具有目标发音人音色和韵律的语音二进制数据

接口说明

• 当前支持通过 WebSocket 协议在线调用

• 请求内容包括：

  • payload字段为将请求参数序列化后的json文本
  • data字段为将音频二进制文件按照base64格式编码（标准base64，RFC 4648）的文本

• 使用备注：

公共参数

参考详细说明功能调用-通用协议-WebSocket

配置参数

payload配置参数为json字符串格式

示例：

{
    "speaker": "zh_female_qingxin_stream",
    "audio_info": {
        "sample_rate": 16000,
        "channel": 1,
        "format": "s16le",
    },
    "audio_config": {
        "sample_rate": 24000,
        "channel": 1,
        "format": "s16le",
    },
}

响应格式

响应中不同消息类型的接收：

• Text message文本消息类型，包含控制事件和响应数据
• Binary message二进制消息类型，接收只包含二进制数据的响应。
  文本消息类型响应的定义如下：

示例：

{
    "task_id": "***",
    "message_id": "***",
    "namespace": "VoiceConversionStream",
    "event": "TaskFinished",
    "status_code": 20000000,
    "status_text": "OK"
}

参考示例

流式调用方式参考WebSocket公共流式协议功能调用-通用协议-WebSocket。

Golang

// Code sample:
// use websocket client to invoke SAMI Streaming Service
package main

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"sync"
	"time"

	"github.com/gorilla/websocket"
)

type WebSocketRequest struct {
	Token     string  `header:"SAMI-Token,required" json:"token,required" query:"token,required"`
	Appkey    string  `json:"appkey,required" query:"appkey,required" vd:"$!=''"`
	Namespace string  `json:"namespace,required" query:"namespace,required" vd:"$!=''"`
	Version   string  `json:"version,omitempty" query:"version"`
	Event     string  `json:"event,omitempty" query:"event"`
	Payload   *string `form:"payload" json:"payload,omitempty"`
	Data      []byte  `form:"data" json:"data,omitempty"`
	TaskId    string  `json:"task_id,omitempty" query:"task_id"`
}

type WebSocketResponse struct {
	TaskId     string  `form:"task_id,required" json:"task_id,required" query:"task_id,required"`
	MessageId  string  `form:"message_id,required" json:"message_id,required" query:"message_id,required"`
	Namespace  string  `form:"namespace,required" json:"namespace,required" query:"namespace,required"`
	Event      string  `form:"event,required" json:"event,required" query:"event,required"`
	StatusCode int32   `form:"status_code,required" json:"status_code,required" query:"status_code,required"`
	StatusText string  `form:"status_text,required" json:"status_text,required" query:"status_text,required"`
	Payload    *string `form:"payload,omitempty" json:"payload,omitempty" query:"payload,omitempty"`
	Data       []byte  `form:"data,omitempty" json:"data,omitempty" query:"data,omitempty"`
}

const (
	EventStartTask    = "StartTask"
	EventTaskStarted  = "TaskStarted"
	EventFinishTask   = "FinishTask"
	EventTaskFinished = "TaskFinished"
)

var (
	// websocket domain
	addr = flag.String("addr", "sami.bytedance.com", "http service address")
	// user auth
	token  = "your_token"
	appkey = "your_appkey"

	inputAudioPath  = "/path/to/input/audio.wav"
	outputAudioPath = "/path/to/output/audio.wav"

	speaker   = "zh_female_qingxin_stream"
	namespace = "VoiceConversionStream"

	u url.URL
	c *websocket.Conn

	interrupt chan os.Signal
	err       error
)

func main() {
	flag.Parse()
	log.SetFlags(0)

	interrupt = make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	log.Printf("connecting to %s\n", u.String())
	start := time.Now()
	u = url.URL{Scheme: "wss", Host: *addr, Path: "/api/v1/ws"}
	requestHeader := http.Header{}
	c, _, err = websocket.DefaultDialer.Dial(u.String(), requestHeader)
	if err != nil {
		log.Fatal("dial:", err)
	}
	defer c.Close()
	fmt.Printf("connection cost time: %dms\n", time.Since(start).Milliseconds())

	voiceConversion(speaker)
}

func voiceConversion(speaker string) {
	var buf bytes.Buffer
	request := &WebSocketRequest{
		Token:     token,
		Appkey:    appkey,
		Namespace: namespace,
		Event:     EventStartTask,
	}
	inputSampleRate := 16000
	inputChannel := 1
	inputFormat := "s16le"
	outputSampleRate := 24000
	outputChannel := 1
	outputFormat := "s16le"
	testSpeaker := speaker
	voiceConversionPayload := VoiceConversionRequest{
		Speaker: &testSpeaker,
		Info: &AudioInfo{
			SampleRate: inputSampleRate,
			Channel:    inputChannel,
			Format:     inputFormat,
		},
		AudioConfig: &AudioConfig{
			Format:     outputFormat,
			Channel:    outputChannel,
			SampleRate: outputSampleRate,
		},
	}

	b, _ := json.Marshal(&voiceConversionPayload)
	plStr := string(b)
	request.Payload = &plStr
	fmt.Println("req payload: ", *request.Payload)
	controlMsg, _ := json.Marshal(request)

	_ = c.WriteMessage(websocket.TextMessage, controlMsg)
	if err = readTaskStartedEvent(); err != nil {
		return
	}
	testData, err := ioutil.ReadFile(inputAudioPath)
	if err != nil {
		panic(err)
	}
	testData, err = ReadWav2PCM(testData)
	if err != nil {
		panic(err)
	}
	fmt.Println(len(testData))
	for i := 0; i < 0; i++ {
		testData = append(testData, testData...)
	}
	wg := sync.WaitGroup{}
	wg.Add(2)
	start := time.Now()
	// send
	go func() {
		defer wg.Done()
		defer func() {
			request.Event = EventFinishTask
			request.Payload = nil
			controlMsg, _ = json.Marshal(request)
			err := c.WriteMessage(websocket.TextMessage, controlMsg)
			if err != nil {
				log.Println("read err=", err)
				return
			}
			fmt.Println("send end, time: ", time.Since(start))
		}()
		isFirst := true
		l := len(testData)
		step := 3200
		times := int(math.Ceil(float64(l) / float64(step)))
		for i := 0; i < times; i++ {
			var dataToSend []byte
			if (i+1)*step > len(testData) {
				dataToSend = testData[i*step:]
			} else {
				dataToSend = testData[i*step : (i+1)*step]
			}
			if isFirst {
				start = time.Now()
				isFirst = false
			}
			if len(dataToSend) > 0 {
				err := c.WriteMessage(websocket.BinaryMessage, dataToSend)
				if err != nil {
					log.Println("read err=", err)
					return
				}
			}
			fmt.Printf("send len[%v] at %v\n", len(dataToSend), time.Now().Format("2006-01-02T15:04:05.999"))
		}
	}()
	// recv
	go func() {
		defer wg.Done()
		isFirst := true
		for {
			mt, message, err := c.ReadMessage()
			if err != nil {
				log.Println("read err=", err)
				break
			}
			if mt == websocket.BinaryMessage {
				log.Printf("binary, recv: byte[%v] at %v", len(message), time.Now().Format("2006-01-02T15:04:05.999"))
				if isFirst {
					fmt.Println("first resp: ", time.Since(start), time.Now())
					isFirst = false
				}
				buf.Write(message)
			} else {
				if isFirst {
					fmt.Println("first resp: ", time.Since(start), time.Now())
					isFirst = false
				}
				wsResp := WebSocketResponse{}
				err := json.Unmarshal(message, &wsResp)
				if err != nil {
					log.Printf("recv text message, parse failed")
				}
				log.Println(string(message))
				if wsResp.Event == EventTaskFinished {
					log.Printf("recv TaskFinished event: %+v, cost_time=%vms", wsResp, time.Since(start).Milliseconds())
					break
				}
				if wsResp.Payload != nil {
					log.Println(*wsResp.Payload, time.Now())
				}
				buf.Write(wsResp.Data)
			}
		}
	}()
	wg.Wait()
	fmt.Printf("recv len: %v\n", buf.Len())
	wavBinaryData := ConvertPcm2Wav(buf.Bytes(), outputChannel, outputSampleRate, 16)
	if err = ioutil.WriteFile(outputAudioPath, wavBinaryData, 0600); err != nil {
		fmt.Printf("write file %v failed: %v\n", outputAudioPath, err)
	}
	fmt.Println("write file name:", outputAudioPath)
}

func readTaskStartedEvent() error {
	msgType, message, err := c.ReadMessage()
	if err != nil {
		log.Println("read TaskStarted event failed, ", err)
		return err
	}
	if msgType != websocket.TextMessage {
		log.Println("read TaskStarted event failed, message type not TextMessage: ", msgType)
		return fmt.Errorf("MessageTypeNotMatch")
	}
	taskStartedEvent := &WebSocketResponse{}
	err = json.Unmarshal(message, taskStartedEvent)
	if err != nil {
		log.Printf("Unmarshal failed, err=%v", err)
		return err
	}
	if taskStartedEvent.Event != EventTaskStarted {
		log.Printf("read TaskStarted event failed, event type not match: %+v", *taskStartedEvent)
		return fmt.Errorf("EventTypeNotMatch")
	}
	log.Printf("read TaskStarted event: %+v", *taskStartedEvent)
	return nil
}

type VoiceConversionRequest struct {
	Info        *AudioInfo   `json:"audio_info,omitempty"`
	AudioConfig *AudioConfig `json:"audio_config,omitempty"`
	Speaker     *string      `json:"speaker,omitempty"`
}

type AudioInfo struct {
	SampleRate int    `json:"sample_rate,omitempty"`
	Channel    int    `json:"channel,omitempty"`
	Format     string `json:"format,omitempty"`
}

type AudioConfig struct {
	Format     string `json:"format,omitempty"`
	SampleRate int    `json:"sample_rate,omitempty"` // default 24000, [8000, 16000, 22050, 24000, 32000, 44100, 48000]
	Channel    int    `json:"channel,omitempty"`     // 1, 2
}

// ConvertPcm2Wav Add wav file header, convert pcm data to wav format
func ConvertPcm2Wav(inData []byte, channels int, sampleRate int, bitsPerSample int) []byte {
	if inData == nil {
		return nil
	}
	dataSize := len(inData)
	wavHeader := make([]byte, 44)
	offset := 0
	// RIFF
	copy(wavHeader[offset:offset+4], "RIFF")
	offset += 4
	// File size
	binary.LittleEndian.PutUint32(wavHeader[offset:offset+4], uint32(dataSize+36))
	offset += 4
	// File type
	copy(wavHeader[offset:offset+4], "WAVE")
	offset += 4
	// Format chunk marker
	copy(wavHeader[offset:offset+4], "fmt ")
	offset += 4
	// Length of above format data
	binary.LittleEndian.PutUint32(wavHeader[offset:offset+4], uint32(16))
	offset += 4
	// Format type(1 for PCM)
	binary.LittleEndian.PutUint16(wavHeader[offset:offset+2], uint16(1))
	offset += 2
	// Channel num
	binary.LittleEndian.PutUint16(wavHeader[offset:offset+2], uint16(channels))
	offset += 2
	// Sample rate
	binary.LittleEndian.PutUint32(wavHeader[offset:offset+4], uint32(sampleRate))
	offset += 4
	// Byte rate
	binary.LittleEndian.PutUint32(wavHeader[offset:offset+4], uint32(sampleRate*channels*bitsPerSample/8))
	offset += 4
	// BlockAlign
	binary.LittleEndian.PutUint16(wavHeader[offset:offset+2], uint16(channels*bitsPerSample/8))
	offset += 2
	// Bits per sample
	binary.LittleEndian.PutUint16(wavHeader[offset:offset+2], uint16(bitsPerSample))
	offset += 2
	// "data" Chunk marker
	copy(wavHeader[offset:offset+4], "data")
	offset += 4
	// PCM data size
	binary.LittleEndian.PutUint32(wavHeader[offset:offset+4], uint32(dataSize))
	return append(wavHeader, inData...)
}

func ReadWav2PCM(wavData []byte) ([]byte, error) {
	if wavData == nil {
		return nil, nil
	}
	offset := 36
	wavlen := len(wavData)
	for offset < wavlen-4 {
		if string(wavData[offset:offset+4]) == "data" {
			return wavData[offset+8:], nil
		} else {
			offset += 2
		}
	}
	err := errors.New(fmt.Sprintf("pcm data not found in wav data, data length : %v", len(wavData)))
	return nil, err
}

Python

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# author:bytedance

import asyncio
import base64
import json
import time
import wave

import websockets

appkey = "your_appkey"
token = "your_token"

namespace = "VoiceConversionStream"
speaker = "zh_female_qingxin_stream"

# 注意输入输出音频编码等需要满足接口要求(具体要求参考接入文档)
input_audio_path = "/path/to/input/audio"
output_audio_path = "/path/to/output/audio"

# 注意可输出的音频编码等信息参考接入文档
result_channel = 1
result_sample_width = 2
result_sample_rate = 24000


async def vc_recv_ws(ws):
    # try:
    await asyncio.sleep(0.1)
    st = time.perf_counter()
    flag = 0
    first_package_time = None
    wave_data = wave.open(output_audio_path, "wb")
    wave_data.setnchannels(result_channel)
    wave_data.setsampwidth(result_sample_width)
    wave_data.setframerate(result_sample_rate)
    while True:
        res = await ws.recv()
        try:
            if isinstance(res, str):
                print("receive text message, ", end="")
                res_dict = json.loads(res)
                if "data" in res_dict:
                    if flag == 0:
                        first_package_time = time.perf_counter() - st
                        flag = 1
                    wave_data.writeframes(res_dict["data"])
                    print(" data=byte[%d]" % len(res_dict["data"]), end="")
                if "payload" in res_dict:
                    print(" payload=%s" % res_dict["payload"], end="")
                print(" task_id=%s, event=%s status_code=%d status_text=%s" % (
                    res_dict["task_id"], res_dict["event"], res_dict["status_code"], res_dict["status_text"]))
                if res_dict["status_code"] != 0 and res_dict["status_code"] != 20000000:
                    print("task failed: ", res_dict)
                    await ws.close()
                    break
                if res_dict["event"] == "TaskFinished":
                    await ws.close()
                    break
            else:
                print("receive binary message, len=%d" % len(res))
                wave_data.writeframes(res)
                if flag == 0:
                    first_package_time = time.perf_counter() - st
                    flag = 1
                # print(res)
        except Exception as e:
            print("exception", e)
            break
    if first_package_time is not None:
        print("首包时间：", first_package_time)
    wave_data.close()


async def vc_send_ws(ws, req, payload):
    with open(input_audio_path, "rb") as input_file:
        input_data = input_file.read()
    send_batch = int(0.1 * payload["audio_info"]["sample_rate"] * payload["audio_info"]["channel"] * 16 / 8)
    start = 0
    while True:
        try:
            end = start + send_batch if start + send_batch < len(input_data) else len(input_data)
            req["data"] = base64.b64encode(input_data[start:end]).decode()
            await ws.send(json.dumps(req))
            print("send len", len(req["data"]))
            await asyncio.sleep(0.1)
        except Exception as e:
            print("exception", e)
            break
        if end >= len(input_data):
            break
        start = end
    req["event"] = "FinishTask"
    await ws.send(json.dumps(req))


async def vc_read_task_started(ws) -> bool:
    res = await ws.recv()
    try:
        if isinstance(res, str):
            print("receive text message, ", end="")
            res_dict = json.loads(res)
            print("res dict", res_dict)
            if "event" in res_dict:
                event = res_dict["event"]
                if event == "TaskStarted":
                    return True
            return False
        else:
            print("receive binary message, len=%d" % len(res))
            return False
    except Exception as e:
        print("exception", e)
        return False


async def vc_ws():
    api_url = "ws://sami.bytedance.com/api/v1/ws"
    async with websockets.connect(api_url) as ws:
        payload = {
            "speaker": speaker,
            "audio_info": {
                "format": "s16le",
                "sample_rate": 16000,
                "channel": 1,
            },
            "audio_config": {
                "format": "s16le",
                "sample_rate": result_sample_rate,
                "channel": result_channel
            }}
        req = {
            "token": token,
            "appkey": appkey,
            "namespace": namespace,
            "event": "StartTask",
            "payload": json.dumps(payload),
        }
        await ws.send(json.dumps(req))
        res = await vc_read_task_started(ws)
        if not res:
            print("hand shake error")
            return
        send, recv = asyncio.create_task(vc_send_ws(ws, req, payload)), asyncio.create_task(vc_recv_ws(ws))
        await send
        await recv


if __name__ == '__main__':
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    tasks = [asyncio.ensure_future(vc_ws(), loop=loop) for i in range(1)]
    loop.run_until_complete(asyncio.wait(tasks))

常见问题

• 通用FAQ

• 通用状态码

• 特定错误码：

附录

发音人列表

备注：当前仅上线中文发音人，其他语种敬请期待。

中文发音人