Fine Tuning XTTS V2 with (forked) Coqui

Fine tuning XTTS v2 with the forked Coqui project. Coqui AI shut down earlier this year, so what does that mean for us? Here I go over adjusting the Coqui XTTS v2 training recipe, creating a dataset using Audacity and faster-whisper, and training a single speaker and multispeaker XTTS v2 english model

Convert wavs to single channel, 22050hz

for /f "tokens=*" %a in ('dir /b *.wav') do ffmpeg -i "%a" -vn -ar 22050 -ac 1 converted_wavs\%~na.wav

Transcribe clips (Updated 6/27/2024)

from faster_whisper import WhisperModel
import os


#model_size = "large-v3"
model_size = "distil-large-v3"
# Define directory containing audio files
audio_dir = "c:\\voicemodel\\mixed-more\\"
wav_dir = audio_dir + "wavs\\"
model = WhisperModel(model_size, device="cuda", compute_type="float16")
def transcribe_and_save(audio_file):

    base_filename, _ = os.path.splitext(os.path.basename(audio_file))
    try:
        segments, _ = model.transcribe(audio_file, beam_size=5, language="en", condition_on_previous_text=False)
        transcript = " ".join(segment.text.lstrip() for segment in segments)
        print(f"Transcribed and saved: {audio_file}")
        with open(os.path.join(audio_dir, "metadata.csv"), "a", encoding="utf-8") as csvfile:
            csvfile.write(f"{base_filename}|{transcript}|{transcript}\n")
    except Exception as e:
        print(f"Error transcribing {audio_file}: {e}")
for filename in os.listdir(wav_dir):
    if filename.endswith(".wav"):
        audio_file = os.path.join(wav_dir, filename)
        transcribe_and_save(audio_file)

Install Coqui

git clone https://github.com/idiap/coqui-ai-TTS tts
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
cd tts
pip install -e .[all,dev,docs]

Install faster-whisper

conda create -n faster-whisper python=3.10 git pip
conda activate faster-whisper
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
pip install --force-reinstall "faster-whisper @ https://github.com/SYSTRAN/faster-whisper/archive/refs/heads/master.tar.gz"

Remove optimizer

import os
import glob
import torch
model_dir = "./run/training/GPT_XTTS_v2.0_LJSpeech_FT-bw-June-25-2024_11+37AM-98c0f86c/"
model_path = model_dir + "best_model.pth"
checkpoint = torch.load(model_path, map_location=torch.device("cpu"))
del checkpoint["optimizer"]
#https://github.com/coqui-ai/TTS/discussions/3474#discussioncomment-7965683
for key in list(checkpoint["model"].keys()):
    if "dvae" in key:
        del checkpoint["model"][key]
torch.save(checkpoint, model_dir+"model_small.pth")

Example inference test

import os
import torch
import torchaudio
from datetime import datetime
from TTS.tts.configs.xtts_config import XttsConfig
from TTS.tts.models.xtts import Xtts
import logging
import time
logger = logging.getLogger(__name__)
print("Loading model...")
config = XttsConfig()
config.load_json("./xtts_44800/config.json")
model = Xtts.init_from_config(config)
model.load_checkpoint(config, checkpoint_dir="./xtts_44800/", use_deepspeed=False)
model.cuda()
speakerpath = "./speakers/"
phrases = ["I like big butts and I cannot lie, You other brothers can't deny. That when a girl walks in with an itty bitty waist, And a round thing in your face, you get sprung. Wanna pull up tough 'cause you notice that butt was stuffed. Deep in the jeans she's wearin', I'm hooked and I can't stop starin'. Oh, baby, I wanna get with ya, And take your picture, My homeboys tried to warn me, But that butt you got makes Me-me so horny.","X T T S is very sensitive to noise and outlying samples in the dataset."]
print(len(phrases))
#phrase_one = "X T T S is very sensitive to noise and outlying samples in the dataset."
for filename in os.listdir(speakerpath):
    if filename.endswith(".wav"):
        for phrase in phrases:
            start_time = time.time()

            print("Computing speaker latents...")
            gpt_cond_latent, speaker_embedding = model.get_conditioning_latents(audio_path=[speakerpath+filename])

            print("Inference...")
            out = model.inference(
            phrase,
            "en",
            gpt_cond_latent, 
            speaker_embedding,
            temperature=0.7, # Add custom parameters here
        )
            now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
            # compute stats
            process_time = time.time() - start_time
            audio_time = len(torch.tensor(out["wav"]).unsqueeze(0) / 24000)
            logger.warning("Processing time: %.3f", process_time)
            logger.warning("Real-time factor: %.3f", process_time / audio_time)
            torchaudio.save(f"{now}-xtts.wav", torch.tensor(out["wav"]).unsqueeze(0), 24000)