From 1239c6735c3928412278b8fa44e140a3ff72ced6 Mon Sep 17 00:00:00 2001 From: Dave Niewinski Date: Mon, 11 Dec 2023 12:00:57 -0500 Subject: [PATCH] Added riva setup for ROS --- glados_riva/CMakeLists.txt | 16 ++++ glados_riva/package.xml | 14 +++ glados_riva/scripts/.gitignore | 1 + glados_riva/scripts/glados_talk.py | 119 ++++++++++++++++++++++++++ glados_riva/scripts/transcribe_mic.py | 97 +++++++++++++++++++++ 5 files changed, 247 insertions(+) create mode 100644 glados_riva/CMakeLists.txt create mode 100644 glados_riva/package.xml create mode 100644 glados_riva/scripts/.gitignore create mode 100644 glados_riva/scripts/glados_talk.py create mode 100644 glados_riva/scripts/transcribe_mic.py diff --git a/glados_riva/CMakeLists.txt b/glados_riva/CMakeLists.txt new file mode 100644 index 0000000..d502e45 --- /dev/null +++ b/glados_riva/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.0.2) +project(glados_riva) + +find_package(catkin REQUIRED COMPONENTS) + +catkin_package( +# INCLUDE_DIRS include +# LIBRARIES glados_riva +# CATKIN_DEPENDS rospy std_msgs +# DEPENDS system_lib +) + +include_directories( +# include + ${catkin_INCLUDE_DIRS} +) diff --git a/glados_riva/package.xml b/glados_riva/package.xml new file mode 100644 index 0000000..38fedce --- /dev/null +++ b/glados_riva/package.xml @@ -0,0 +1,14 @@ + + + glados_riva + 0.0.0 + The glados_riva package + + Dave Niewinski + + CC-BY-3 + + catkin + rospy + std_msgs + diff --git a/glados_riva/scripts/.gitignore b/glados_riva/scripts/.gitignore new file mode 100644 index 0000000..d8dd753 --- /dev/null +++ b/glados_riva/scripts/.gitignore @@ -0,0 +1 @@ +*.wav diff --git a/glados_riva/scripts/glados_talk.py b/glados_riva/scripts/glados_talk.py new file mode 100644 index 0000000..be8b989 --- /dev/null +++ b/glados_riva/scripts/glados_talk.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT + +import argparse +import time +import wave +from pathlib import Path +import string +import riva.client +from riva.client.argparse_utils import add_connection_argparse_parameters +import rospy +from std_msgs.msg import String + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="A speech synthesis via Riva AI Services. You HAVE TO provide at least one of arguments " + "`--output`, `--play-audio`, `--list-devices`, `--output-device`.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--voice", + help="A voice name to use. If this parameter is missing, then the server will try a first available model " + "based on parameter `--language-code`.", + ) + parser.add_argument("-o", "--output", type=Path, help="Output file .wav file to write synthesized audio.") + parser.add_argument( + "--play-audio", + action="store_true", + help="Whether to play input audio simultaneously with transcribing. If `--output-device` is not provided, " + "then the default output audio device will be used.", + ) + parser.add_argument("--list-devices", action="store_true", help="List output audio devices indices.") + parser.add_argument("--output-device", type=int, help="Output device to use.") + parser.add_argument("--language-code", default='en-US', help="A language of input text.") + parser.add_argument( + "--sample-rate-hz", type=int, default=44100, help="Number of audio frames per second in synthesized audio.") + parser.add_argument( + "--stream", + action="store_true", + help="If this option is set, then streaming synthesis is applied. Streaming means that audio is yielded " + "as it gets ready. If `--stream` is not set, then a synthesized audio is returned in 1 response only when " + "all text is processed.", + ) + parser = add_connection_argparse_parameters(parser) + args = parser.parse_args() + if args.output is None and not args.play_audio and args.output_device is None and not args.list_devices: + parser.error( + f"You have to provide at least one of arguments: `--play-audio`, `--output-device`, `--output`, " + f"`--list-devices`." + ) + if args.output is not None: + args.output = args.output.expanduser() + if args.list_devices or args.output_device or args.play_audio: + import riva.client.audio_io + return args + +def callback(msg): + global TTS + TTS = msg.data + +def main() -> None: + global TTS + TTS = None + + args = parse_args() + if args.list_devices: + riva.client.audio_io.list_output_devices() + return + + rospy.init_node('riva_tts', anonymous=True) + rospy.Subscriber("speak", String, callback) + + auth = riva.client.Auth(args.ssl_cert, args.use_ssl, args.server) + service = riva.client.SpeechSynthesisService(auth) + nchannels = 1 + sampwidth = 2 + sound_stream, out_f = None, None + try: + if args.output_device is not None or args.play_audio: + sound_stream = riva.client.audio_io.SoundCallBack( + args.output_device, nchannels=nchannels, sampwidth=sampwidth, framerate=args.sample_rate_hz + ) + + rate = rospy.Rate(2) + while not rospy.is_shutdown(): + if TTS != None: + text = TTS + + rospy.loginfo("Generating audio for request...") + rospy.loginfo(text) + start = time.time() + + resp = service.synthesize(text, args.voice, args.language_code, sample_rate_hz=args.sample_rate_hz) + stop = time.time() + rospy.loginfo("Time spent: " + str(stop - start) + "s") + + if sound_stream is not None: + sound_stream(resp.audio) + + filename = text.translate(str.maketrans('', '', string.punctuation)) + ".wav" + filename = filename.replace(" ", "_") + out_f = wave.open(filename, 'wb') + out_f.setnchannels(nchannels) + out_f.setsampwidth(sampwidth) + out_f.setframerate(args.sample_rate_hz) + out_f.writeframesraw(resp.audio) + out_f.close() + + TTS = None + + rate.sleep() + finally: + if sound_stream is not None: + sound_stream.close() + + +if __name__ == '__main__': + main() diff --git a/glados_riva/scripts/transcribe_mic.py b/glados_riva/scripts/transcribe_mic.py new file mode 100644 index 0000000..b75b740 --- /dev/null +++ b/glados_riva/scripts/transcribe_mic.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT + +import argparse + +import riva.client +from riva.client.argparse_utils import add_asr_config_argparse_parameters, add_connection_argparse_parameters + +import riva.client.audio_io + +import rospy +from std_msgs.msg import String + +def parse_args() -> argparse.Namespace: + default_device_info = riva.client.audio_io.get_default_input_device_info() + default_device_index = None if default_device_info is None else default_device_info['index'] + parser = argparse.ArgumentParser( + description="Streaming transcription from microphone via Riva AI Services", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--input-device", type=int, default=default_device_index, help="An input audio device to use.") + parser.add_argument("--list-devices", action="store_true", help="List input audio device indices.") + parser = add_asr_config_argparse_parameters(parser, profanity_filter=True) + parser = add_connection_argparse_parameters(parser) + parser.add_argument( + "--sample-rate-hz", + type=int, + help="A number of frames per second in audio streamed from a microphone.", + default=16000, + ) + parser.add_argument( + "--file-streaming-chunk", + type=int, + default=1600, + help="A maximum number of frames in a audio chunk sent to server.", + ) + args = parser.parse_args() + return args + + +def main() -> None: + args = parse_args() + if args.list_devices: + riva.client.audio_io.list_input_devices() + return + + rospy.init_node('riva_asr', anonymous=True) + + inter_pub = rospy.Publisher('intermediate', String, queue_size=10) + final_pub = rospy.Publisher('final', String, queue_size=10) + + auth = riva.client.Auth(args.ssl_cert, args.use_ssl, args.server) + asr_service = riva.client.ASRService(auth) + config = riva.client.StreamingRecognitionConfig( + config=riva.client.RecognitionConfig( + encoding=riva.client.AudioEncoding.LINEAR_PCM, + language_code=args.language_code, + max_alternatives=1, + profanity_filter=args.profanity_filter, + enable_automatic_punctuation=args.automatic_punctuation, + verbatim_transcripts=not args.no_verbatim_transcripts, + sample_rate_hertz=args.sample_rate_hz, + audio_channel_count=1, + ), + interim_results=True, + ) + + riva.client.add_word_boosting_to_config(config, args.boosted_lm_words, args.boosted_lm_score) + with riva.client.audio_io.MicrophoneStream( + args.sample_rate_hz, + args.file_streaming_chunk, + device=args.input_device, + ) as audio_chunk_iterator: + responses = asr_service.streaming_response_generator(audio_chunks=audio_chunk_iterator, streaming_config=config) + + for response in responses: + if not response.results: + continue + partial_transcript = "" + for result in response.results: + if not result.alternatives: + continue + transcript = result.alternatives[0].transcript + + if result.is_final: + for i, alternative in enumerate(result.alternatives): + final_pub.publish(alternative.transcript) + + else: + partial_transcript += transcript + + if partial_transcript: + inter_pub.publish(partial_transcript) + +if __name__ == '__main__': + main()