mirror of
https://github.com/davesarmoury/GLaDOS.git
synced 2025-09-26 22:31:26 +08:00
Updated to create manifest for NeMo retraining
This commit is contained in:
parent
689c9ed154
commit
19a60e54f0
1
.gitignore
vendored
1
.gitignore
vendored
@ -1 +1,2 @@
|
||||
manifest.json
|
||||
*.wav
|
||||
|
@ -5,6 +5,10 @@ from multiprocessing import cpu_count
|
||||
from multiprocessing.pool import ThreadPool
|
||||
import shutil
|
||||
import os
|
||||
from bs4 import BeautifulSoup
|
||||
import soundfile as sf
|
||||
import string
|
||||
import json
|
||||
|
||||
class bcolors:
|
||||
HEADER = '\033[95m'
|
||||
@ -17,15 +21,24 @@ class bcolors:
|
||||
BOLD = '\033[1m'
|
||||
UNDERLINE = '\033[4m'
|
||||
|
||||
blocklist = ["potato", "_ding_"]
|
||||
|
||||
blocklist = ["potato", "_ding_", "00_part1_entry-6"]
|
||||
audio_dir = 'audio'
|
||||
download_threads = 64
|
||||
|
||||
def prep():
|
||||
if os.path.exists(audio_dir):
|
||||
print("Deleting previously downloaded audio")
|
||||
shutil.rmtree(audio_dir)
|
||||
|
||||
os.mkdir(audio_dir)
|
||||
|
||||
def remove_punctuation(str):
|
||||
return str.translate(str.maketrans('', '', string.punctuation))
|
||||
|
||||
def audio_duration(fn):
|
||||
f = sf.SoundFile(fn)
|
||||
return f.frames / f.samplerate
|
||||
|
||||
def download_file(args):
|
||||
url, filename = args[0], args[1]
|
||||
|
||||
@ -37,8 +50,7 @@ def download_file(args):
|
||||
return filename, False
|
||||
|
||||
def download_parallel(args):
|
||||
cpus = cpu_count()
|
||||
results = ThreadPool(cpus - 1).imap_unordered(download_file, args)
|
||||
results = ThreadPool(download_threads).imap_unordered(download_file, args)
|
||||
for result in results:
|
||||
if result[1]:
|
||||
print(bcolors.OKGREEN + "[" + u'\u2713' + "] " + bcolors.ENDC + result[0])
|
||||
@ -46,28 +58,58 @@ def download_parallel(args):
|
||||
print(bcolors.FAIL + "[" + u'\u2715' + "] " + bcolors.ENDC + result[0])
|
||||
|
||||
def main():
|
||||
global blocklist
|
||||
r = requests.get("https://theportalwiki.com/wiki/GLaDOS_voice_lines")
|
||||
|
||||
split_text = r.text.split()
|
||||
urls = []
|
||||
filenames = []
|
||||
texts = []
|
||||
|
||||
for chunk in split_text:
|
||||
if "https:" in chunk and ".wav" in chunk:
|
||||
url = chunk.replace('"', '').replace("href=", '')
|
||||
soup = BeautifulSoup(r.text, 'html.parser')
|
||||
for link_item in soup.find_all('a'):
|
||||
url = link_item.get("href", None)
|
||||
if url:
|
||||
if "https:" in url and ".wav" in url:
|
||||
list_item = link_item.find_parent("li")
|
||||
ital_item = list_item.find_all('i')
|
||||
if ital_item:
|
||||
text = ital_item[0].text
|
||||
text = text.replace('"', '')
|
||||
filename = url[url.rindex("/")+1:]
|
||||
|
||||
if "[" not in text and "]" not in text:
|
||||
if url not in urls:
|
||||
for s in blocklist:
|
||||
if s in url:
|
||||
break
|
||||
else:
|
||||
urls.append(url)
|
||||
filenames.append(url[url.rindex("/")+1:])
|
||||
filenames.append(filename)
|
||||
texts.append(text)
|
||||
|
||||
print("Found " + str(len(urls)) + " urls")
|
||||
|
||||
args = zip(urls, filenames)
|
||||
|
||||
download_parallel(args)
|
||||
#prep()
|
||||
#download_parallel(args)
|
||||
|
||||
#{"audio_filepath": "audio/nada_lily_21_haggard_0316.wav",
|
||||
#"text": "awake ye kings",
|
||||
#"duration": 1.3,
|
||||
#"text_no_preprocessing": "\u201cAwake, ye kings,\u201d",
|
||||
#"text_normalized": "\"Awake, ye kings,\""}
|
||||
outFile=open(os.path.join(audio_dir, "manifest.json"), 'w')
|
||||
for i in range(len(urls)):
|
||||
item = {}
|
||||
text = texts[i]
|
||||
filename = filenames[i]
|
||||
item["audio_filepath"] = filename
|
||||
item["text_normalized"] = text
|
||||
item["text_no_preprocessing"] = text
|
||||
item["text"] = text.lower()
|
||||
item["duration"] = audio_duration(os.path.join(audio_dir, filename))
|
||||
outFile.write(json.dumps(item, ensure_ascii=True, sort_keys=True) + "\n")
|
||||
|
||||
outFile.close()
|
||||
|
||||
main()
|
@ -1 +1,3 @@
|
||||
beautifulsoup4
|
||||
requests
|
||||
soundfile
|
||||
|
Loading…
Reference in New Issue
Block a user