Add JSON, SRT, and WebVTT ASR exports

README.md

@@ -85,6 +85,19 @@ $ riva_asr_client --audio_file audio_folder
 
 Note that only single-channel audio files in the `.wav` format are currently supported.
 
+Final transcripts can be exported as structured JSON, SubRip, or WebVTT by choosing the
+corresponding extension for `--output_filename`:
+```
+$ riva_asr_client --audio_file audio.wav --output_filename transcript.json
+$ riva_asr_client --audio_file audio.wav --output_filename transcript.srt
+$ riva_streaming_asr_client --audio_file audio.wav --output_filename transcript.vtt
+```
+The format can also be selected explicitly with `--output_format=json|srt|vtt`. JSON output uses
+one object per recognition request (JSON Lines when processing multiple requests). SRT and VTT
+automatically request word timestamps and currently require one audio file with
+`--num_iterations=1`; microphone input is not supported for subtitle export. Each final recognition
+result becomes one subtitle cue.
+
 Other options and information can be found by running the clients with `-help`
 
 ### Speech Synthesis (TTS) Client

riva/clients/asr/BUILD

@@ -27,6 +27,16 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "transcript_exporter",
+    srcs = ["transcript_exporter.cc"],
+    hdrs = ["transcript_exporter.h"],
+    deps = [
+        ":asr_client_helper",
+        "@nvriva_common//riva/proto:riva_grpc_asr",
+    ],
+)
+
 cc_library(
     name = "client_call",
     srcs = ["client_call.h", "client_call.cc"],
@@ -39,6 +49,7 @@ cc_library(
         ],
     }) + [
         ":asr_client_helper",
+        ":transcript_exporter",
         "@com_github_grpc_grpc//:grpc++",
         "@nvriva_common//riva/proto:riva_grpc_asr",
         "@glog//:glog",
@@ -54,6 +65,7 @@ cc_library(
     deps = [
         ":asr_client_helper",
         ":client_call",
+        ":transcript_exporter",
         "//riva/utils/wav:reader",
         "//riva/utils/opus",
         "@glog//:glog",
@@ -79,6 +91,7 @@ cc_binary(
     deps = [
         ":asr_client_helper",
         ":client_call",
+        ":transcript_exporter",
         "@nvriva_common//riva/proto:riva_grpc_asr",
         "//riva/utils:stamping",
         "//riva/utils/files:files",
@@ -97,6 +110,7 @@ cc_binary(
         ":asr_client_helper",
         ":client_call",
         ":streaming_recognize_client",
+        ":transcript_exporter",
         "@nvriva_common//riva/proto:riva_grpc_asr",
         "//riva/utils/files:files",
         "//riva/utils/wav:reader",
@@ -126,3 +140,13 @@ cc_test(
     ],
     tags = ["needs_alsa"]
 )
+
+cc_test(
+    name = "transcript_exporter_test",
+    srcs = ["transcript_exporter_test.cc"],
+    deps = [
+        ":transcript_exporter",
+        "@googletest//:gtest_main",
+        "@nvriva_common//riva/proto:riva_grpc_asr",
+    ],
+)

riva/clients/asr/client_call.cc

@@ -43,6 +43,13 @@ ClientCall::AppendResult(const nr_asr::StreamingRecognitionResult& result)
     bool is_final = result.is_final();
     if (is_final) {
       int num_alternatives = result.alternatives_size();
+      if (num_alternatives > 0) {
+        Results::Segment segment;
+        for (int a = 0; a < num_alternatives; ++a) {
+          segment.alternatives.push_back(result.alternatives(a));
+        }
+        latest_result_.segments.push_back(std::move(segment));
+      }
       latest_result_.final_transcripts.resize(num_alternatives);
       latest_result_.final_scores.resize(num_alternatives);
       latest_result_.final_time_stamps.resize(num_alternatives);
@@ -88,7 +95,7 @@ ClientCall::AppendResult(const nr_asr::StreamingRecognitionResult& result)
 }
 
 void
-ClientCall::PrintResult(bool audio_device, std::ofstream& output_file)
+ClientCall::PrintResult(bool audio_device)
 {
   std::cout << "-----------------------------------------------------------" << std::endl;
 
@@ -100,16 +107,8 @@ ClientCall::PrintResult(bool audio_device, std::ofstream& output_file)
 
   std::cout << std::endl;
   std::cout << "Final transcripts: " << std::endl;
-  if (latest_result_.final_transcripts.size() == 0) {
-    output_file << "{\"audio_filepath\": \"" << filename << "\",";
-    output_file << "\"text\": \"\"}" << std::endl;
-  } else {
+  if (latest_result_.final_transcripts.size() > 0) {
     for (uint32_t a = 0; a < latest_result_.final_transcripts.size(); ++a) {
-      if (a == 0) {
-        output_file << "{\"audio_filepath\": \"" << filename << "\",";
-        output_file << "\"text\": \"" << EscapeTranscript(latest_result_.final_transcripts[a])
-                    << "\"}" << std::endl;
-      }
       std::cout << a << " : " << latest_result_.final_transcripts[a]
                 << latest_result_.partial_transcript << std::endl;
       std::cout << std::endl;

riva/clients/asr/client_call.h

@@ -41,7 +41,7 @@ class ClientCall {
 
   void AppendResult(const nr_asr::StreamingRecognitionResult& result);
 
-  void PrintResult(bool audio_device, std::ofstream& output_file);
+  void PrintResult(bool audio_device);
 
   // Container for the data we expect from the server.
   nr_asr::StreamingRecognizeResponse response;

riva/clients/asr/riva_asr_client.cc

@@ -28,6 +28,7 @@
 #include "riva/utils/stamping.h"
 #include "riva/utils/wav/wav_reader.h"
 #include "riva_asr_client_helper.h"
+#include "transcript_exporter.h"
 
 using grpc::Status;
 using grpc::StatusCode;
@@ -51,6 +52,9 @@ DEFINE_int32(num_iterations, 1, "Number of times to loop over audio files");
 DEFINE_int32(num_parallel_requests, 10, "Number of parallel requests to keep in flight");
 DEFINE_bool(print_transcripts, true, "Print final transcripts");
 DEFINE_string(output_filename, "", "Filename to write output transcripts");
+DEFINE_string(
+    output_format, "",
+    "Transcript output format: json, srt, or vtt. By default, infer from output_filename");
 DEFINE_string(model_name, "", "Name of the TRTIS model to use");
 DEFINE_bool(list_models, false, "List available models on server");
 DEFINE_bool(output_ctm, false, "If true, output format should be NIST CTM");
@@ -100,7 +104,8 @@ class RecognizeClient {
       const std::string& boosted_phrases_file, float boosted_phrases_score,
       bool speaker_diarization, int32_t diarization_max_speakers, int32_t start_history,
       float start_threshold, int32_t stop_history, int32_t stop_history_eou, float stop_threshold,
-      float stop_threshold_eou, std::string custom_configuration)
+      float stop_threshold_eou, std::string custom_configuration,
+      TranscriptOutputFormat output_format)
       : stub_(nr_asr::RivaSpeechRecognition::NewStub(channel)), language_code_(language_code),
         max_alternatives_(max_alternatives), profanity_filter_(profanity_filter),
         word_time_offsets_(word_time_offsets), automatic_punctuation_(automatic_punctuation),
@@ -113,14 +118,15 @@ class RecognizeClient {
         start_history_(start_history), start_threshold_(start_threshold),
         stop_history_(stop_history), stop_history_eou_(stop_history_eou),
         stop_threshold_(stop_threshold), stop_threshold_eou_(stop_threshold_eou),
-        custom_configuration_(custom_configuration)
+        custom_configuration_(custom_configuration), output_format_(output_format), cue_index_(1)
   {
     if (!output_filename.empty()) {
       output_file_.open(output_filename);
       if (ctm) {
         write_fn_ = &RecognizeClient::WriteCTM;
       } else {
-        write_fn_ = &RecognizeClient::WriteJSON;
+        write_fn_ = &RecognizeClient::WriteTranscript;
+        InitializeTranscriptOutput(output_file_, output_format_);
       }
     }
 
@@ -164,19 +170,13 @@ class RecognizeClient {
     }
   }
 
-  void WriteJSON(const Results& result, const std::string& filename)
+  void WriteTranscript(const Results& result, const std::string& filename)
   {
-    if (result.final_transcripts.size() == 0) {
-      output_file_ << "{\"audio_filepath\": \"" << filename << "\",";
-      output_file_ << "\"text\": \"\"}" << std::endl;
-    } else {
-      for (size_t a = 0; a < result.final_transcripts.size(); ++a) {
-        if (a == 0) {
-          output_file_ << "{\"audio_filepath\": \"" << filename << "\",";
-          output_file_ << "\"text\": \"" << EscapeTranscript(result.final_transcripts.at(a))
-                       << "\"}" << std::endl;
-        }
-      }
+    std::string error;
+    if (!WriteTranscriptOutput(
+            output_file_, output_format_, result, filename, &cue_index_, &error)) {
+      std::cerr << "Unable to export transcript: " << error << std::endl;
+      num_failed_requests_++;
     }
   }
 
@@ -439,6 +439,8 @@ class RecognizeClient {
   float stop_threshold_;
   float stop_threshold_eou_;
   std::string custom_configuration_;
+  TranscriptOutputFormat output_format_;
+  size_t cue_index_;
 };
 
 int
@@ -459,6 +461,7 @@ main(int argc, char** argv)
   str_usage << "           --num_parallel_requests=<integer> " << std::endl;
   str_usage << "           --print_transcripts=<true|false> " << std::endl;
   str_usage << "           --output_filename=<string>" << std::endl;
+  str_usage << "           --output_format=<json|srt|vtt>" << std::endl;
   str_usage << "           --output-ctm=<true|false>" << std::endl;
   str_usage << "           --verbatim_transcripts=<true|false>" << std::endl;
   str_usage << "           --language_code=<bcp 47 language code (such as en-US), optional>" << std::endl;
@@ -501,6 +504,23 @@ main(int argc, char** argv)
     return 1;
   }
 
+  if (FLAGS_output_ctm && !FLAGS_output_format.empty()) {
+    std::cerr << "output_ctm and output_format cannot be used together." << std::endl;
+    return 1;
+  }
+  TranscriptOutputFormat output_format = TranscriptOutputFormat::kJson;
+  if (!FLAGS_output_filename.empty() && !FLAGS_output_ctm) {
+    std::string error;
+    if (!ParseTranscriptOutputFormat(
+            FLAGS_output_filename, FLAGS_output_format, &output_format, &error)) {
+      std::cerr << error << std::endl;
+      return 1;
+    }
+    if (TranscriptOutputRequiresWordTimeOffsets(output_format)) {
+      FLAGS_word_time_offsets = true;
+    }
+  }
+
   bool flag_set = gflags::GetCommandLineFlagInfoOrDie("riva_uri").is_default;
   const char* riva_uri = getenv("RIVA_URI");
 
@@ -541,15 +561,6 @@ main(int argc, char** argv)
     return 0;
   }
 
-  RecognizeClient recognize_client(
-      grpc_channel, FLAGS_language_code, FLAGS_max_alternatives, FLAGS_profanity_filter,
-      FLAGS_word_time_offsets, FLAGS_automatic_punctuation,
-      /* separate_recognition_per_channel*/ false, FLAGS_print_transcripts, FLAGS_output_filename,
-      FLAGS_model_name, FLAGS_output_ctm, FLAGS_verbatim_transcripts, FLAGS_boosted_words_file,
-      (float)FLAGS_boosted_words_score, FLAGS_speaker_diarization, FLAGS_diarization_max_speakers,
-      FLAGS_start_history, FLAGS_start_threshold, FLAGS_stop_history, FLAGS_stop_history_eou,
-      FLAGS_stop_threshold, FLAGS_stop_threshold_eou, FLAGS_custom_configuration);
-
   // Preload all wav files, sort by size to reduce tail effects
   std::vector<std::shared_ptr<WaveData>> all_wav;
   try {
@@ -563,6 +574,20 @@ main(int argc, char** argv)
     std::cout << "No audio files specified. Exiting." << std::endl;
     return 1;
   }
+  if (TranscriptOutputRequiresWordTimeOffsets(output_format) &&
+      (all_wav.size() != 1 || FLAGS_num_iterations != 1)) {
+    std::cerr << "SRT and VTT export require one audio file and num_iterations=1." << std::endl;
+    return 1;
+  }
+
+  RecognizeClient recognize_client(
+      grpc_channel, FLAGS_language_code, FLAGS_max_alternatives, FLAGS_profanity_filter,
+      FLAGS_word_time_offsets, FLAGS_automatic_punctuation,
+      /* separate_recognition_per_channel*/ false, FLAGS_print_transcripts, FLAGS_output_filename,
+      FLAGS_model_name, FLAGS_output_ctm, FLAGS_verbatim_transcripts, FLAGS_boosted_words_file,
+      (float)FLAGS_boosted_words_score, FLAGS_speaker_diarization, FLAGS_diarization_max_speakers,
+      FLAGS_start_history, FLAGS_start_threshold, FLAGS_stop_history, FLAGS_stop_history_eou,
+      FLAGS_stop_threshold, FLAGS_stop_threshold_eou, FLAGS_custom_configuration, output_format);
 
   uint32_t all_wav_max = all_wav.size() * FLAGS_num_iterations;
   std::vector<std::shared_ptr<WaveData>> all_wav_repeated;
@@ -614,5 +639,5 @@ main(int argc, char** argv)
     }
   }
 
-  return 0;
+  return recognize_client.NumFailedRequests() ? 1 : 0;
 }

riva/clients/asr/riva_asr_client_helper.cc

@@ -125,6 +125,13 @@ AppendResult(
   }
 
   int num_alternatives = result.alternatives_size();
+  if (num_alternatives > 0) {
+    Results::Segment segment;
+    for (int a = 0; a < num_alternatives; ++a) {
+      segment.alternatives.push_back(result.alternatives(a));
+    }
+    output_result.segments.push_back(std::move(segment));
+  }
   output_result.final_transcripts.resize(num_alternatives);
   output_result.final_scores.resize(num_alternatives);
   output_result.final_time_stamps.resize(num_alternatives);
@@ -244,4 +251,4 @@ ReadCustomConfiguration(std::string& custom_configuration)
     }
   }
   return custom_configuration_map;
-}
+}

riva/clients/asr/riva_asr_client_helper.h

@@ -37,6 +37,10 @@ std::string static inline EscapeTranscript(const std::string& input_str)
 }
 
 struct Results {
+  struct Segment {
+    std::vector<nr_asr::SpeechRecognitionAlternative> alternatives;
+  };
+
   std::vector<std::string> final_transcripts;
   std::vector<float> final_scores;
   std::string partial_transcript;
@@ -45,6 +49,7 @@ struct Results {
   std::vector<nr_asr::WordInfo> partial_time_stamps;
   int request_cnt;
   float audio_processed;
+  std::vector<Segment> segments;
 };
 
 void AppendResult(

riva/clients/asr/riva_streaming_asr_client.cc

@@ -61,7 +61,10 @@ DEFINE_bool(print_transcripts, true, "Print final transcripts");
 DEFINE_bool(interim_results, true, "Print intermediate transcripts");
 DEFINE_string(
     output_filename, "final_transcripts.json",
-    "Filename of .json file containing output transcripts");
+    "Filename containing output transcripts");
+DEFINE_string(
+    output_format, "",
+    "Transcript output format: json, srt, or vtt. By default, infer from output_filename");
 DEFINE_string(model_name, "", "Name of the TRTIS model to use");
 DEFINE_bool(list_models, false, "List available models on server");
 DEFINE_string(language_code, "", "Language code of the model to use");
@@ -137,6 +140,7 @@ main(int argc, char** argv)
   str_usage << "           --num_parallel_requests=<integer> " << std::endl;
   str_usage << "           --print_transcripts=<true|false> " << std::endl;
   str_usage << "           --output_filename=<string>" << std::endl;
+  str_usage << "           --output_format=<json|srt|vtt>" << std::endl;
   str_usage << "           --verbatim_transcripts=<true|false>" << std::endl;
   str_usage << "           --language_code=<bcp 47 language code (such as en-US), optional>" << std::endl;
   str_usage << "           --boosted_words_file=<string>" << std::endl;
@@ -179,6 +183,21 @@ main(int argc, char** argv)
     return 1;
   }
 
+  TranscriptOutputFormat output_format;
+  std::string output_error;
+  if (!ParseTranscriptOutputFormat(
+          FLAGS_output_filename, FLAGS_output_format, &output_format, &output_error)) {
+    std::cerr << output_error << std::endl;
+    return 1;
+  }
+  if (TranscriptOutputRequiresWordTimeOffsets(output_format)) {
+    FLAGS_word_time_offsets = true;
+    if (!FLAGS_audio_device.empty()) {
+      std::cerr << "SRT and VTT export currently support audio_file input only." << std::endl;
+      return 1;
+    }
+  }
+
   bool flag_set = gflags::GetCommandLineFlagInfoOrDie("riva_uri").is_default;
   const char* riva_uri = getenv("RIVA_URI");
 
@@ -227,7 +246,7 @@ main(int argc, char** argv)
       FLAGS_verbatim_transcripts, FLAGS_boosted_words_file, FLAGS_boosted_words_score,
       FLAGS_start_history, FLAGS_start_threshold, FLAGS_stop_history, FLAGS_stop_history_eou,
       FLAGS_stop_threshold, FLAGS_stop_threshold_eou, FLAGS_custom_configuration,
-      FLAGS_speaker_diarization, FLAGS_diarization_max_speakers);
+      FLAGS_speaker_diarization, FLAGS_diarization_max_speakers, output_format);
 
   if (FLAGS_audio_file.size()) {
     return recognize_client.DoStreamingFromFile(