mirror of
https://github.com/davesarmoury/GLaDOS.git
synced 2025-11-04 14:49:45 +08:00
Added TAO intermediate notebook
This commit is contained in:
parent
f5b3ebb0f6
commit
89b90e7346
401
TrainGLaDOS_NeMo.ipynb
Normal file
401
TrainGLaDOS_NeMo.ipynb
Normal file
@ -0,0 +1,401 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b276bbe0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#!sudo apt install sox libsndfile1 ffmpeg\n",
|
||||
"#!pip3 install wget unidecode pynini==2.1.4\n",
|
||||
"#!pip3 install git+https://github.com/NVIDIA/NeMo.git@v1.12.0#egg=nemo_toolkit[all]\n",
|
||||
"#!wget https://raw.githubusercontent.com/NVIDIA/NeMo/main/nemo_text_processing/install_pynini.sh\n",
|
||||
"#!bash install_pynini.sh"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c3b7b7e3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"from multiprocessing import cpu_count\n",
|
||||
"from multiprocessing.pool import ThreadPool\n",
|
||||
"import shutil\n",
|
||||
"import os\n",
|
||||
"from bs4 import BeautifulSoup\n",
|
||||
"import soundfile as sf\n",
|
||||
"import string\n",
|
||||
"import json\n",
|
||||
"import re\n",
|
||||
"import num2words\n",
|
||||
"\n",
|
||||
"class bcolors:\n",
|
||||
" HEADER = '\\033[95m'\n",
|
||||
" OKBLUE = '\\033[94m'\n",
|
||||
" OKCYAN = '\\033[96m'\n",
|
||||
" OKGREEN = '\\033[92m'\n",
|
||||
" WARNING = '\\033[93m'\n",
|
||||
" FAIL = '\\033[91m'\n",
|
||||
" ENDC = '\\033[0m'\n",
|
||||
" BOLD = '\\033[1m'\n",
|
||||
" UNDERLINE = '\\033[4m'\n",
|
||||
"\n",
|
||||
"blocklist = [\"potato\", \"_ding_\", \"00_part1_entry-6\"]\n",
|
||||
"audio_dir = 'audio'\n",
|
||||
"download_threads = 64\n",
|
||||
"\n",
|
||||
"def prep(args, overwrite=False):\n",
|
||||
" already_exists = os.path.exists(audio_dir)\n",
|
||||
" \n",
|
||||
" if already_exists and not overwrite:\n",
|
||||
" print(\"Data already downloaded\")\n",
|
||||
" return\n",
|
||||
" \n",
|
||||
" if already_exists:\n",
|
||||
" print(\"Deleting previously downloaded audio\")\n",
|
||||
" shutil.rmtree(audio_dir)\n",
|
||||
"\n",
|
||||
" os.mkdir(audio_dir)\n",
|
||||
" download_parallel(args)\n",
|
||||
"\n",
|
||||
"def remove_punctuation(str):\n",
|
||||
" return str.translate(str.maketrans('', '', string.punctuation))\n",
|
||||
" \n",
|
||||
"def audio_duration(fn):\n",
|
||||
" f = sf.SoundFile(fn)\n",
|
||||
" return f.frames / f.samplerate\n",
|
||||
"\n",
|
||||
"def download_file(args):\n",
|
||||
" url, filename = args[0], args[1]\n",
|
||||
"\n",
|
||||
" try:\n",
|
||||
" response = requests.get(url)\n",
|
||||
" open(os.path.join(audio_dir, filename), \"wb\").write(response.content)\n",
|
||||
" return filename, True\n",
|
||||
" except:\n",
|
||||
" return filename, False\n",
|
||||
"\n",
|
||||
"def download_parallel(args):\n",
|
||||
" results = ThreadPool(download_threads).imap_unordered(download_file, args)\n",
|
||||
" for result in results:\n",
|
||||
" if result[1]:\n",
|
||||
" print(bcolors.OKGREEN + \"[\" + u'\\u2713' + \"] \" + bcolors.ENDC + result[0])\n",
|
||||
" else:\n",
|
||||
" print(bcolors.FAIL + \"[\" + u'\\u2715' + \"] \" + bcolors.ENDC + result[0])\n",
|
||||
"\n",
|
||||
"def main():\n",
|
||||
" r = requests.get(\"https://theportalwiki.com/wiki/GLaDOS_voice_lines\")\n",
|
||||
"\n",
|
||||
" urls = []\n",
|
||||
" filenames = []\n",
|
||||
" texts = []\n",
|
||||
"\n",
|
||||
" soup = BeautifulSoup(r.text.encode('utf-8').decode('ascii', 'ignore'), 'html.parser')\n",
|
||||
" for link_item in soup.find_all('a'):\n",
|
||||
" url = link_item.get(\"href\", None)\n",
|
||||
" if url:\n",
|
||||
" if \"https:\" in url and \".wav\" in url:\n",
|
||||
" list_item = link_item.find_parent(\"li\")\n",
|
||||
" ital_item = list_item.find_all('i')\n",
|
||||
" if ital_item:\n",
|
||||
" text = ital_item[0].text\n",
|
||||
" text = text.replace('\"', '')\n",
|
||||
" filename = url[url.rindex(\"/\")+1:]\n",
|
||||
"\n",
|
||||
" if \"[\" not in text and \"]\" not in text and \"$\" not in text:\n",
|
||||
" if url not in urls:\n",
|
||||
" for s in blocklist:\n",
|
||||
" if s in url:\n",
|
||||
" break\n",
|
||||
" else:\n",
|
||||
" urls.append(url)\n",
|
||||
" filenames.append(filename)\n",
|
||||
" text = text.replace('*', '')\n",
|
||||
" text = re.sub(r\"(\\d+)\", lambda x: num2words.num2words(int(x.group(0))), text)\n",
|
||||
" texts.append(text)\n",
|
||||
"\n",
|
||||
" print(\"Found \" + str(len(urls)) + \" urls\")\n",
|
||||
"\n",
|
||||
" args = zip(urls, filenames)\n",
|
||||
"\n",
|
||||
" prep(args)\n",
|
||||
" \n",
|
||||
" total_audio_time = 0\n",
|
||||
" outFile=open(os.path.join(audio_dir, \"manifest.json\"), 'w')\n",
|
||||
" for i in range(len(urls)):\n",
|
||||
" item = {}\n",
|
||||
" text = texts[i]\n",
|
||||
" filename = filenames[i]\n",
|
||||
" item[\"audio_filepath\"] = os.path.join(audio_dir, filename)\n",
|
||||
" #item[\"text_normalized\"] = text\n",
|
||||
" #item[\"text_no_preprocessing\"] = text\n",
|
||||
" item[\"text\"] = text.lower()\n",
|
||||
" item[\"duration\"] = audio_duration(os.path.join(audio_dir, filename))\n",
|
||||
" total_audio_time = total_audio_time + item[\"duration\"]\n",
|
||||
" outFile.write(json.dumps(item, ensure_ascii=True, sort_keys=True) + \"\\n\")\n",
|
||||
" \n",
|
||||
" outFile.close()\n",
|
||||
" print(str(total_audio_time/60.0) + \" min\")\n",
|
||||
"\n",
|
||||
"main()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "82a7d268",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!head -n 1 ./audio/manifest.json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "832fcd67",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!cat ./audio/manifest.json | tail -n 2 > ./manifest_validation.json\n",
|
||||
"!cat ./audio/manifest.json | head -n -2 > ./manifest_train.json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e56749bf",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"home_path = !(echo $HOME)\n",
|
||||
"home_path = home_path[0]\n",
|
||||
"print(home_path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9bf1fb18",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"import IPython.display as ipd\n",
|
||||
"from matplotlib.pyplot import imshow\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"\n",
|
||||
"from nemo.collections.tts.models import FastPitchModel\n",
|
||||
"FastPitchModel.from_pretrained(\"tts_en_fastpitch\")\n",
|
||||
"\n",
|
||||
"from pathlib import Path\n",
|
||||
"nemo_files = [p for p in Path(f\"{home_path}/.cache/torch/NeMo/\").glob(\"**/tts_en_fastpitch_align.nemo\")]\n",
|
||||
"print(f\"Copying {nemo_files[0]} to ./\")\n",
|
||||
"Path(\"./tts_en_fastpitch_align.nemo\").write_bytes(nemo_files[0].read_bytes())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "81741e30",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!wget https://raw.githubusercontent.com/nvidia/NeMo/v1.12.0/examples/tts/fastpitch_finetune.py\n",
|
||||
"!wget https://raw.githubusercontent.com/NVIDIA/NeMo/v1.12.0/examples/tts/hifigan_finetune.py\n",
|
||||
" \n",
|
||||
"!mkdir -p conf\n",
|
||||
"!cd conf \\\n",
|
||||
"&& wget https://raw.githubusercontent.com/nvidia/NeMo/v1.12.0/examples/tts/conf/fastpitch_align_v1.05.yaml \\\n",
|
||||
"&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/v1.12.0/examples/tts/conf/hifigan/hifigan.yaml \\\n",
|
||||
"&& cd .."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "64b51e94",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# additional files\n",
|
||||
"!mkdir -p tts_dataset_files && cd tts_dataset_files \\\n",
|
||||
"&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/v1.12.0/scripts/tts_dataset_files/cmudict-0.7b_nv22.08 \\\n",
|
||||
"&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/v1.12.0/scripts/tts_dataset_files/heteronyms-052722 \\\n",
|
||||
"&& wget https://raw.githubusercontent.com/NVIDIA/NeMo/v1.12.0/nemo_text_processing/text_normalization/en/data/whitelist/lj_speech.tsv \\\n",
|
||||
"&& cd .."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6b92a469",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!(python3 fastpitch_finetune.py --config-name=fastpitch_align_v1.05.yaml \\\n",
|
||||
" train_dataset=./manifest_train.json \\\n",
|
||||
" validation_datasets=./manifest_validation.json \\\n",
|
||||
" sup_data_path=./fastpitch_sup_data \\\n",
|
||||
" phoneme_dict_path=tts_dataset_files/cmudict-0.7b_nv22.08 \\\n",
|
||||
" heteronyms_path=tts_dataset_files/heteronyms-052722 \\\n",
|
||||
" whitelist_path=tts_dataset_files/lj_speech.tsv \\\n",
|
||||
" exp_manager.exp_dir=./glados_out \\\n",
|
||||
" +init_from_nemo_model=./tts_en_fastpitch_align.nemo \\\n",
|
||||
" trainer.max_epochs=100 \\\n",
|
||||
" trainer.check_val_every_n_epoch=25 \\\n",
|
||||
" model.train_ds.dataloader_params.batch_size=12 model.validation_ds.dataloader_params.batch_size=12 \\\n",
|
||||
" model.n_speakers=1 model.pitch_mean=121.9 model.pitch_std=23.1 \\\n",
|
||||
" model.pitch_fmin=30 model.pitch_fmax=512 model.optim.lr=2e-4 \\\n",
|
||||
" ~model.optim.sched model.optim.name=adam trainer.devices=1 trainer.strategy=null \\\n",
|
||||
" +model.text_tokenizer.add_blank_at=true \\\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "780dba9a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from nemo.collections.tts.models import HifiGanModel\n",
|
||||
"from nemo.collections.tts.models import FastPitchModel\n",
|
||||
"\n",
|
||||
"vocoder = HifiGanModel.from_pretrained(\"tts_hifigan\")\n",
|
||||
"vocoder = vocoder.eval().cuda()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "831239e2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def infer(spec_gen_model, vocoder_model, str_input, speaker=None):\n",
|
||||
" \"\"\"\n",
|
||||
" Synthesizes spectrogram and audio from a text string given a spectrogram synthesis and vocoder model.\n",
|
||||
" \n",
|
||||
" Args:\n",
|
||||
" spec_gen_model: Spectrogram generator model (FastPitch in our case)\n",
|
||||
" vocoder_model: Vocoder model (HiFiGAN in our case)\n",
|
||||
" str_input: Text input for the synthesis\n",
|
||||
" speaker: Speaker ID\n",
|
||||
" \n",
|
||||
" Returns:\n",
|
||||
" spectrogram and waveform of the synthesized audio.\n",
|
||||
" \"\"\"\n",
|
||||
" with torch.no_grad():\n",
|
||||
" parsed = spec_gen_model.parse(str_input)\n",
|
||||
" if speaker is not None:\n",
|
||||
" speaker = torch.tensor([speaker]).long().to(device=spec_gen_model.device)\n",
|
||||
" spectrogram = spec_gen_model.generate_spectrogram(tokens=parsed, speaker=speaker)\n",
|
||||
" audio = vocoder_model.convert_spectrogram_to_audio(spec=spectrogram)\n",
|
||||
" \n",
|
||||
" if spectrogram is not None:\n",
|
||||
" if isinstance(spectrogram, torch.Tensor):\n",
|
||||
" spectrogram = spectrogram.to('cpu').numpy()\n",
|
||||
" if len(spectrogram.shape) == 3:\n",
|
||||
" spectrogram = spectrogram[0]\n",
|
||||
" if isinstance(audio, torch.Tensor):\n",
|
||||
" audio = audio.to('cpu').numpy()\n",
|
||||
" return spectrogram, audio\n",
|
||||
"\n",
|
||||
"def get_best_ckpt_from_last_run(\n",
|
||||
" base_dir, \n",
|
||||
" new_speaker_id, \n",
|
||||
" duration_mins, \n",
|
||||
" mixing_enabled, \n",
|
||||
" original_speaker_id, \n",
|
||||
" model_name=\"FastPitch\"\n",
|
||||
" ): \n",
|
||||
" mixing = \"no_mixing\" if not mixing_enabled else \"mixing\"\n",
|
||||
" \n",
|
||||
" d = \"glados_out\"\n",
|
||||
" \n",
|
||||
" exp_dirs = list([i for i in (Path(base_dir) / d / model_name).iterdir() if i.is_dir()])\n",
|
||||
" last_exp_dir = sorted(exp_dirs)[-1]\n",
|
||||
" \n",
|
||||
" last_checkpoint_dir = last_exp_dir / \"checkpoints\"\n",
|
||||
" \n",
|
||||
" last_ckpt = list(last_checkpoint_dir.glob('*-last.ckpt'))\n",
|
||||
"\n",
|
||||
" if len(last_ckpt) == 0:\n",
|
||||
" raise ValueError(f\"There is no last checkpoint in {last_checkpoint_dir}.\")\n",
|
||||
" \n",
|
||||
" return str(last_ckpt[0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2fad0610",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"new_speaker_id = 6097\n",
|
||||
"duration_mins = 5\n",
|
||||
"mixing = False\n",
|
||||
"original_speaker_id = \"ljspeech\"\n",
|
||||
"\n",
|
||||
"last_ckpt = get_best_ckpt_from_last_run(\"./\", new_speaker_id, duration_mins, mixing, original_speaker_id)\n",
|
||||
"print(last_ckpt)\n",
|
||||
"\n",
|
||||
"spec_model = FastPitchModel.load_from_checkpoint(last_ckpt)\n",
|
||||
"spec_model.eval().cuda()\n",
|
||||
"\n",
|
||||
"# Only need to set speaker_id if there is more than one speaker\n",
|
||||
"speaker_id = None\n",
|
||||
"if mixing:\n",
|
||||
" speaker_id = 1\n",
|
||||
"\n",
|
||||
"num_val = 2 # Number of validation samples\n",
|
||||
"val_records = []\n",
|
||||
"with open(\"manifest_validation.json\", \"r\") as f:\n",
|
||||
" for i, line in enumerate(f):\n",
|
||||
" val_records.append(json.loads(line))\n",
|
||||
" if len(val_records) >= num_val:\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
"for val_record in val_records:\n",
|
||||
" print(\"Real validation audio\")\n",
|
||||
" ipd.display(ipd.Audio(val_record['audio_filepath'], rate=22050))\n",
|
||||
" print(f\"SYNTHESIZED FOR -- Speaker: {new_speaker_id} | Dataset size: {duration_mins} mins | Mixing:{mixing} | Text: {val_record['text']}\")\n",
|
||||
" spec, audio = infer(spec_model, vocoder, val_record['text'], speaker=speaker_id)\n",
|
||||
" ipd.display(ipd.Audio(audio, rate=22050))\n",
|
||||
" %matplotlib inline\n",
|
||||
" imshow(spec, origin=\"lower\", aspect=\"auto\")\n",
|
||||
" plt.show()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.10"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
1259
TrainGLaDOS_TAO.ipynb
Normal file
1259
TrainGLaDOS_TAO.ipynb
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
from multiprocessing import cpu_count
|
||||
from multiprocessing.pool import ThreadPool
|
||||
@ -9,6 +7,8 @@ from bs4 import BeautifulSoup
|
||||
import soundfile as sf
|
||||
import string
|
||||
import json
|
||||
import re
|
||||
import num2words
|
||||
|
||||
class bcolors:
|
||||
HEADER = '\033[95m'
|
||||
@ -25,12 +25,19 @@ blocklist = ["potato", "_ding_", "00_part1_entry-6"]
|
||||
audio_dir = 'audio'
|
||||
download_threads = 64
|
||||
|
||||
def prep():
|
||||
if os.path.exists(audio_dir):
|
||||
def prep(args, overwrite=False):
|
||||
already_exists = os.path.exists(audio_dir)
|
||||
|
||||
if already_exists and not overwrite:
|
||||
print("Data already downloaded")
|
||||
return
|
||||
|
||||
if already_exists:
|
||||
print("Deleting previously downloaded audio")
|
||||
shutil.rmtree(audio_dir)
|
||||
|
||||
os.mkdir(audio_dir)
|
||||
download_parallel(args)
|
||||
|
||||
def remove_punctuation(str):
|
||||
return str.translate(str.maketrans('', '', string.punctuation))
|
||||
@ -64,7 +71,7 @@ def main():
|
||||
filenames = []
|
||||
texts = []
|
||||
|
||||
soup = BeautifulSoup(r.text, 'html.parser')
|
||||
soup = BeautifulSoup(r.text.encode('utf-8').decode('ascii', 'ignore'), 'html.parser')
|
||||
for link_item in soup.find_all('a'):
|
||||
url = link_item.get("href", None)
|
||||
if url:
|
||||
@ -84,32 +91,37 @@ def main():
|
||||
else:
|
||||
urls.append(url)
|
||||
filenames.append(filename)
|
||||
text = text.replace('*', '')
|
||||
text = re.sub(r"(\d+)", lambda x: num2words.num2words(int(x.group(0))), text)
|
||||
texts.append(text)
|
||||
|
||||
print("Found " + str(len(urls)) + " urls")
|
||||
|
||||
args = zip(urls, filenames)
|
||||
|
||||
prep()
|
||||
download_parallel(args)
|
||||
prep(args)
|
||||
|
||||
#{"audio_filepath": "audio/nada_lily_21_haggard_0316.wav",
|
||||
#"text": "awake ye kings",
|
||||
#"duration": 1.3,
|
||||
#"text_no_preprocessing": "\u201cAwake, ye kings,\u201d",
|
||||
#"text_normalized": "\"Awake, ye kings,\""}
|
||||
|
||||
total_audio_time = 0
|
||||
outFile=open(os.path.join(audio_dir, "manifest.json"), 'w')
|
||||
for i in range(len(urls)):
|
||||
item = {}
|
||||
text = texts[i]
|
||||
filename = filenames[i]
|
||||
item["audio_filepath"] = filename
|
||||
item["audio_filepath"] = os.path.join(audio_dir, filename)
|
||||
item["text_normalized"] = text
|
||||
item["text_no_preprocessing"] = text
|
||||
item["text"] = text.lower()
|
||||
item["duration"] = audio_duration(os.path.join(audio_dir, filename))
|
||||
total_audio_time = total_audio_time + item["duration"]
|
||||
outFile.write(json.dumps(item, ensure_ascii=True, sort_keys=True) + "\n")
|
||||
|
||||
outFile.close()
|
||||
print(str(total_audio_time/60.0) + " min")
|
||||
|
||||
main()
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user