292 lines
9.4 KiB
Python
292 lines
9.4 KiB
Python
"""Alibaba CosyVoice backend for narration audio generation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
from urllib import error, request
|
|
|
|
from tts_backends.backend_common import (
|
|
download_audio,
|
|
extension_from_format,
|
|
get_bytes,
|
|
post_json,
|
|
publish_audio_subtitle,
|
|
read_api_key,
|
|
read_http_error,
|
|
)
|
|
from tts_backends.backend_edge import (
|
|
DEFAULT_SUBTITLE_MAX_CHARS,
|
|
format_word_timed_srt,
|
|
)
|
|
|
|
|
|
DEFAULT_ENDPOINT = "https://dashscope.aliyuncs.com/api/v1/services/audio/tts/SpeechSynthesizer"
|
|
DEFAULT_MODEL = "cosyvoice-v3-flash"
|
|
_TICKS_PER_MILLISECOND = 10_000
|
|
|
|
|
|
def output_extension(audio_format: str) -> str:
|
|
return extension_from_format(audio_format)
|
|
|
|
|
|
def read_cosyvoice_api_key(env_name: str) -> str:
|
|
env_names = tuple(dict.fromkeys([env_name, "COSYVOICE_API_KEY", "DASHSCOPE_API_KEY"]))
|
|
return read_api_key(*env_names, label="CosyVoice/DashScope")
|
|
|
|
|
|
def resolve_url(base_url: str | None = None) -> str:
|
|
base = (base_url or os.environ.get("COSYVOICE_TTS_BASE_URL") or DEFAULT_ENDPOINT).rstrip("/")
|
|
if base.endswith("/SpeechSynthesizer"):
|
|
return base
|
|
return base + "/api/v1/services/audio/tts/SpeechSynthesizer"
|
|
|
|
|
|
def _parse_sse_event(data_lines: list[str]) -> dict | None:
|
|
payload = "\n".join(data_lines).strip()
|
|
if not payload or payload == "[DONE]":
|
|
return None
|
|
try:
|
|
event = json.loads(payload)
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError("CosyVoice streaming response contains invalid JSON") from exc
|
|
if not isinstance(event, dict):
|
|
raise RuntimeError("CosyVoice streaming event is not an object")
|
|
return event
|
|
|
|
|
|
def _parse_sse(raw: bytes) -> list[dict]:
|
|
try:
|
|
body = raw.decode("utf-8-sig")
|
|
except UnicodeError as exc:
|
|
raise RuntimeError("CosyVoice streaming response is not valid UTF-8") from exc
|
|
|
|
if not any(line.startswith("data:") for line in body.splitlines()):
|
|
event = _parse_sse_event([body])
|
|
return [event] if event is not None else []
|
|
|
|
events: list[dict] = []
|
|
data_lines: list[str] = []
|
|
for line in body.splitlines():
|
|
if not line:
|
|
event = _parse_sse_event(data_lines)
|
|
if event is not None:
|
|
events.append(event)
|
|
data_lines = []
|
|
continue
|
|
if line.startswith("data:"):
|
|
data_lines.append(line[5:].lstrip())
|
|
event = _parse_sse_event(data_lines)
|
|
if event is not None:
|
|
events.append(event)
|
|
if not events:
|
|
raise RuntimeError("CosyVoice streaming response contains no events")
|
|
return events
|
|
|
|
|
|
def _post_streaming(
|
|
url: str,
|
|
*,
|
|
api_key: str,
|
|
payload: dict,
|
|
) -> list[dict]:
|
|
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
req = request.Request(
|
|
url,
|
|
data=body,
|
|
headers={
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": "application/json",
|
|
"X-DashScope-SSE": "enable",
|
|
},
|
|
method="POST",
|
|
)
|
|
try:
|
|
with request.urlopen(req, timeout=180) as response:
|
|
return _parse_sse(response.read())
|
|
except error.HTTPError as exc:
|
|
raise RuntimeError(read_http_error(exc)) from exc
|
|
except error.URLError as exc:
|
|
raise RuntimeError(
|
|
f"CosyVoice streaming request failed: {exc.reason}"
|
|
) from exc
|
|
|
|
|
|
def _ticks(value: object, *, field: str) -> int:
|
|
if isinstance(value, bool):
|
|
raise RuntimeError(f"CosyVoice word {field} is not numeric")
|
|
try:
|
|
numeric = float(value)
|
|
except (TypeError, ValueError) as exc:
|
|
raise RuntimeError(f"CosyVoice word {field} is not numeric") from exc
|
|
if not math.isfinite(numeric):
|
|
raise RuntimeError(f"CosyVoice word {field} is not finite")
|
|
return int(round(numeric * _TICKS_PER_MILLISECOND))
|
|
|
|
|
|
def _word_boundaries(events: list[dict]) -> tuple[list[dict], str]:
|
|
sentence_text: dict[int, str] = {}
|
|
sentence_words: dict[int, list] = {}
|
|
audio_url = ""
|
|
|
|
for event in events:
|
|
code = event.get("code")
|
|
message = event.get("message")
|
|
if code or message:
|
|
details = ": ".join(str(value) for value in (code, message) if value)
|
|
raise RuntimeError(f"CosyVoice TTS failed: {details}")
|
|
output = event.get("output") or {}
|
|
if not isinstance(output, dict):
|
|
continue
|
|
audio = output.get("audio") or {}
|
|
if isinstance(audio, dict) and isinstance(audio.get("url"), str):
|
|
audio_url = audio["url"]
|
|
|
|
sentence = output.get("sentence") or {}
|
|
if not isinstance(sentence, dict):
|
|
continue
|
|
index = sentence.get("index")
|
|
if isinstance(index, bool) or not isinstance(index, int) or index < 0:
|
|
continue
|
|
original_text = output.get("original_text")
|
|
if isinstance(original_text, str) and original_text:
|
|
sentence_text[index] = original_text
|
|
words = sentence.get("words")
|
|
if isinstance(words, list) and words:
|
|
sentence_words[index] = words
|
|
|
|
if not audio_url:
|
|
raise RuntimeError("CosyVoice streaming response contains no audio URL")
|
|
if not sentence_words:
|
|
raise RuntimeError(
|
|
"CosyVoice returned no word timestamps. Use a cloned voice from "
|
|
"cosyvoice-v3.5-plus/flash, cosyvoice-v3-plus/flash, or cosyvoice-v2, "
|
|
"or a system voice marked as timestamp-supported; otherwise pass "
|
|
"--cosyvoice-audio-only."
|
|
)
|
|
|
|
boundaries: list[dict] = []
|
|
for sentence_index in sorted(sentence_words):
|
|
original_text = sentence_text.get(sentence_index, "")
|
|
for item in sentence_words[sentence_index]:
|
|
if not isinstance(item, dict):
|
|
raise RuntimeError("CosyVoice word timing is not an object")
|
|
word = item.get("text")
|
|
begin_index = item.get("begin_index")
|
|
end_index = item.get("end_index")
|
|
if (
|
|
original_text
|
|
and isinstance(begin_index, int)
|
|
and not isinstance(begin_index, bool)
|
|
and isinstance(end_index, int)
|
|
and not isinstance(end_index, bool)
|
|
and 0 <= begin_index < end_index <= len(original_text)
|
|
):
|
|
word = original_text[begin_index:end_index]
|
|
if not isinstance(word, str) or not word:
|
|
raise RuntimeError("CosyVoice word timing contains no text")
|
|
start = _ticks(item.get("begin_time"), field="start time")
|
|
end = _ticks(item.get("end_time"), field="end time")
|
|
if start < 0 or end <= start:
|
|
raise RuntimeError(
|
|
"CosyVoice response contains an invalid word interval"
|
|
)
|
|
if boundaries and start < boundaries[-1]["offset"]:
|
|
raise RuntimeError(
|
|
"CosyVoice word timestamps are not in chronological order"
|
|
)
|
|
boundaries.append({
|
|
"text": word,
|
|
"offset": start,
|
|
"duration": end - start,
|
|
})
|
|
return boundaries, audio_url
|
|
|
|
|
|
def generate(
|
|
text: str,
|
|
output_path: Path,
|
|
*,
|
|
api_key: str,
|
|
voice_id: str,
|
|
model: str,
|
|
audio_format: str,
|
|
sample_rate: int,
|
|
volume: int | None,
|
|
rate: float | None,
|
|
pitch: float | None,
|
|
instruction: str | None,
|
|
language_hint: str | None,
|
|
base_url: str | None,
|
|
subtitle_path: Path | None = None,
|
|
subtitle_max_chars: int = DEFAULT_SUBTITLE_MAX_CHARS,
|
|
) -> None:
|
|
input_payload: dict[str, object] = {
|
|
"text": text,
|
|
"voice": voice_id,
|
|
"format": audio_format,
|
|
"sample_rate": sample_rate,
|
|
}
|
|
if volume is not None:
|
|
input_payload["volume"] = volume
|
|
if rate is not None:
|
|
input_payload["rate"] = rate
|
|
if pitch is not None:
|
|
input_payload["pitch"] = pitch
|
|
if instruction:
|
|
input_payload["instruction"] = instruction
|
|
if language_hint:
|
|
input_payload["language_hints"] = [language_hint]
|
|
|
|
payload = {
|
|
"model": model,
|
|
"input": input_payload,
|
|
}
|
|
url = resolve_url(base_url)
|
|
|
|
if subtitle_path is not None:
|
|
input_payload["word_timestamp_enabled"] = True
|
|
events = _post_streaming(
|
|
url,
|
|
api_key=api_key,
|
|
payload=payload,
|
|
)
|
|
boundaries, audio_url = _word_boundaries(events)
|
|
subtitle = format_word_timed_srt(
|
|
text,
|
|
boundaries,
|
|
subtitle_max_chars,
|
|
provider_label="CosyVoice",
|
|
)
|
|
publish_audio_subtitle(
|
|
get_bytes(audio_url),
|
|
output_path,
|
|
subtitle,
|
|
subtitle_path,
|
|
)
|
|
return
|
|
|
|
data = post_json(
|
|
url,
|
|
headers={"Authorization": f"Bearer {api_key}"},
|
|
payload=payload,
|
|
timeout=180,
|
|
)
|
|
audio = (data.get("output") or {}).get("audio") or {}
|
|
audio_url = audio.get("url")
|
|
if not audio_url:
|
|
raise RuntimeError(f"CosyVoice response missing audio URL: {data}")
|
|
download_audio(audio_url, output_path)
|
|
|
|
|
|
def print_voices() -> None:
|
|
print("CosyVoice voices are selected by voice.")
|
|
print("Use a system voice name or a cloned/designed voice_id from CosyVoice.")
|
|
print("Timestamp-supported system voice example: longanyang")
|
|
print(
|
|
"For SRT, use a supported CosyVoice cloned voice or a system voice "
|
|
"marked timestamp-supported in the provider catalog."
|
|
)
|