Adds visual output to C++ sample

Bug: 78540198
Change-Id: I26f5f85f0b0e8255b9c7bf641caa8aa0592d13bc

Author: Fleker

Makefile

@@ -75,11 +75,13 @@ CXXFLAGS += $(ALSA_CFLAGS)
 LDFLAGS += $(ALSA_LDFLAGS)
 endif
 
-ASSISTANT_O = $(AUDIO_SRCS:.cc=.o) ./src/audio_input_file.o ./src/json_util.o \
+ASSISTANT_O = $(AUDIO_SRCS:.cc=.o) ./src/audio_input_file.o ./src/base64_encode.o ./src/json_util.o \
 	./src/run_assistant_audio.o ./src/run_assistant_text.o
-ASSISTANT_AUDIO_O = $(AUDIO_SRCS:.cc=.o) ./src/audio_input_file.o ./src/json_util.o \
+ASSISTANT_AUDIO_O = $(AUDIO_SRCS:.cc=.o) ./src/audio_input_file.o ./src/base64_encode.o ./src/json_util.o \
 	./src/run_assistant_audio.o
-ASSISTANT_TEXT_O = ./src/json_util.o ./src/run_assistant_text.o
+ASSISTANT_FILE_O = $(AUDIO_SRCS:.cc=.o) ./src/audio_input_file.o ./src/base64_encode.o ./src/json_util.o \
+	./src/run_assistant_file.o
+ASSISTANT_TEXT_O = ./src/json_util.o ./src/base64_encode.o ./src/run_assistant_text.o
 
 .PHONY: all
 all: run_assistant
@@ -94,7 +96,7 @@ run_assistant_audio: $(GOOGLEAPIS_ASSISTANT_CCS:.cc=.o) googleapis.ar \
 	$(CXX) $^ $(LDFLAGS) -o $@
 
 run_assistant_file: $(GOOGLEAPIS_ASSISTANT_CCS:.cc=.o) googleapis.ar \
-	$(AUDIO_SRCS:.cc=.o) ./src/audio_input_file.o ./src/json_util.o ./src/run_assistant_file.o
+	$(ASSISTANT_FILE_O)
 	$(CXX) $^ $(LDFLAGS) -o $@
 
 run_assistant_text: $(GOOGLEAPIS_ASSISTANT_CCS:.cc=.o) googleapis.ar \

README.md

@@ -113,4 +113,16 @@ To change the locale, include a `locale` parameter:
 echo "Bonjour" | ./run_assistant_text --credentials ./credentials.json --locale "fr-FR"
 ```
 
-Default Assistant gRPC API endpoint is embeddedassistant.googleapis.com. If you want to test with a custom Assistant gRPC API endpoint, you can pass an extra "--api_endpoint CUSTOM_API_ENDPOINT" to run_assistant.
+Default Assistant gRPC API endpoint is `embeddedassistant.googleapis.com`. If you want to test with a custom Assistant gRPC API endpoint, you can pass `--api_endpoint CUSTOM_API_ENDPOINT`.
+
+## Enabling screen output
+
+To get a visual output from the Assistant, provide a command to be run alongside every step of the conversation. It will execute that command along along with a provided argument of a temporary HTML file.
+
+```bash
+echo "what time is it" | ./run_assistant_text --credentials ./credentials.json --html_out google-chrome
+```
+
+After you enter text, it will run `google-chrome /tmp/google-assistant-cpp-screen-out.html`.
+
+If you prefer a different program, use that argument instead.

src/base64_encode.cc

@@ -0,0 +1,39 @@
+/*
+Copyright 2017 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "base64_encode.h"
+
+// https://stackoverflow.com/questions/180947/base64-decode-snippet-in-c by Manuel Martinez
+std::string base64_encode(const std::string &in) {
+  std::string out;
+
+  int val=0, valb=-6;
+  for (u_char c : in) {
+      val = (val<<8) + c;
+      valb += 8;
+      while (valb>=0) {
+          out.push_back("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(val>>valb)&0x3F]);
+          valb-=6;
+      }
+  }
+  if (valb>-6) {
+    out.push_back("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[((val<<8)>>(valb+8))&0x3F]);
+  }
+  while (out.size()%4) {
+    out.push_back('=');
+  }
+  return out;
+}

src/base64_encode.h

@@ -0,0 +1,24 @@
+/*
+Copyright 2018 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef BASE64_ENCODE_H
+#define BASE64_ENCODE_H
+
+#include <string>
+
+std::string base64_encode(const std::string&);
+
+#endif

src/run_assistant_audio.cc

@@ -42,13 +42,15 @@ limitations under the License.
 #include "assistant_config.h"
 #include "audio_input.h"
 #include "audio_input_file.h"
+#include "base64_encode.h"
 #include "json_util.h"
 
 using google::assistant::embedded::v1alpha2::EmbeddedAssistant;
 using google::assistant::embedded::v1alpha2::AssistRequest;
 using google::assistant::embedded::v1alpha2::AssistResponse;
 using google::assistant::embedded::v1alpha2::AudioInConfig;
 using google::assistant::embedded::v1alpha2::AudioOutConfig;
+using google::assistant::embedded::v1alpha2::ScreenOutConfig;
 using google::assistant::embedded::v1alpha2::AssistResponse_EventType_END_OF_UTTERANCE;
 
 using grpc::CallCredentials;
@@ -58,11 +60,10 @@ using grpc::ClientReaderWriter;
 static const std::string kCredentialsTypeUserAccount = "USER_ACCOUNT";
 static const std::string kALSAAudioInput = "ALSA_INPUT";
 static const std::string kLanguageCode = "en-US";
-static const std::string kDeviceInstanceId = "default";
 static const std::string kDeviceModelId = "default";
+static const std::string kDeviceInstanceId = "default";
 
 bool verbose = false;
-bool use_text = false; // Alternatively use text
 
 // Creates a channel to be connected to Google.
 std::shared_ptr<Channel> CreateChannel(const std::string& host) {
@@ -90,24 +91,26 @@ void PrintUsage() {
             << "--credentials <credentials_file> "
             << "[--api_endpoint <API endpoint>] "
             << "[--locale <locale>]"
+            << "[--html_out <command to load HTML page>]"
             << std::endl;
 }
 
 bool GetCommandLineFlags(
     int argc, char** argv, std::string* credentials_file_path,
-    std::string* api_endpoint, std::string* locale) {
+    std::string* api_endpoint, std::string* locale, std::string* html_out_command) {
   const struct option long_options[] = {
     {"credentials", required_argument, nullptr, 'c'},
     {"api_endpoint",     required_argument, nullptr, 'e'},
     {"locale",           required_argument, nullptr, 'l'},
     {"verbose",          no_argument, nullptr, 'v'},
+    {"html_out",         required_argument, nullptr, 'h'},
     {nullptr, 0, nullptr, 0}
   };
   *api_endpoint = ASSISTANT_ENDPOINT;
   while (true) {
     int option_index;
     int option_char =
-        getopt_long(argc, argv, "c:e:l:v", long_options, &option_index);
+        getopt_long(argc, argv, "c:e:l:v:h", long_options, &option_index);
     if (option_char == -1) {
       break;
     }
@@ -124,6 +127,9 @@ bool GetCommandLineFlags(
       case 'v':
         verbose = true;
         break;
+      case 'h':
+        *html_out_command = optarg;
+        break;
       default:
         PrintUsage();
         return false;
@@ -133,7 +139,7 @@ bool GetCommandLineFlags(
 }
 
 int main(int argc, char** argv) {
-  std::string credentials_file_path, api_endpoint, locale;
+  std::string credentials_file_path, api_endpoint, locale, html_out_command;
   #ifndef ENABLE_ALSA
     std::cerr << "ALSA audio input is not supported on this platform."
               << std::endl;
@@ -144,7 +150,7 @@ int main(int argc, char** argv) {
   // https://github.com/grpc/grpc/issues/11366#issuecomment-328595941
   grpc_init();
   if (!GetCommandLineFlags(argc, argv, &credentials_file_path,
-                          &api_endpoint, &locale)) {
+                          &api_endpoint, &locale, &html_out_command)) {
     return -1;
   }
 
@@ -171,6 +177,11 @@ int main(int argc, char** argv) {
       AudioOutConfig::LINEAR16);
     assist_config->mutable_audio_out_config()->set_sample_rate_hertz(16000);
 
+    // Set parameters for screen config
+    assist_config->mutable_screen_out_config()->set_screen_mode(
+      html_out_command.empty() ? ScreenOutConfig::SCREEN_MODE_UNSPECIFIED : ScreenOutConfig::PLAYING
+    );
+
     std::unique_ptr<AudioInput> audio_input;
     // Set the AudioInConfig of the AssistRequest
     assist_config->mutable_audio_in_config()->set_encoding(
@@ -264,11 +275,16 @@ int main(int argc, char** argv) {
                     << ")" << std::endl;
         }
       }
-      if (response.dialog_state_out().supplemental_display_text().size() > 0) {
-        // CUSTOMIZE: render spoken response on screen
-        std::clog << "assistant_sdk response:" << std::endl;
-        std::cout << response.dialog_state_out().supplemental_display_text()
-                  << std::endl;
+      if (!html_out_command.empty() && response.screen_out().data().size() > 0) {
+        std::string html_out_base64 = base64_encode(response.screen_out().data());
+        system((html_out_command + " \"data:text/html;base64, " + html_out_base64 + "\"").c_str());
+      } else if (html_out_command.empty()) {
+        if (response.dialog_state_out().supplemental_display_text().size() > 0) {
+          // CUSTOMIZE: render spoken response on screen
+          std::clog << "assistant_sdk response:" << std::endl;
+          std::cout << response.dialog_state_out().supplemental_display_text()
+                    << std::endl;
+        }
       }
     }
 

src/run_assistant_file.cc

@@ -34,6 +34,7 @@ limitations under the License.
 #include "assistant_config.h"
 #include "audio_input.h"
 #include "audio_input_file.h"
+#include "base64_encode.h"
 #include "json_util.h"
 
 using google::assistant::embedded::v1alpha2::EmbeddedAssistant;
@@ -49,8 +50,8 @@ using grpc::ClientReaderWriter;
 
 static const std::string kCredentialsTypeUserAccount = "USER_ACCOUNT";
 static const std::string kLanguageCode = "en-US";
-static const std::string kDeviceInstanceId = "default";
 static const std::string kDeviceModelId = "default";
+static const std::string kDeviceInstanceId = "default";
 
 bool verbose = false;
 

src/run_assistant_text.cc

@@ -34,13 +34,15 @@ limitations under the License.
 #include "assistant_config.h"
 #include "audio_input.h"
 #include "audio_input_file.h"
+#include "base64_encode.h"
 #include "json_util.h"
 
 using google::assistant::embedded::v1alpha2::EmbeddedAssistant;
 using google::assistant::embedded::v1alpha2::AssistRequest;
 using google::assistant::embedded::v1alpha2::AssistResponse;
 using google::assistant::embedded::v1alpha2::AudioInConfig;
 using google::assistant::embedded::v1alpha2::AudioOutConfig;
+using google::assistant::embedded::v1alpha2::ScreenOutConfig;
 using google::assistant::embedded::v1alpha2::AssistResponse_EventType_END_OF_UTTERANCE;
 
 using grpc::CallCredentials;
@@ -49,8 +51,8 @@ using grpc::ClientReaderWriter;
 
 static const std::string kCredentialsTypeUserAccount = "USER_ACCOUNT";
 static const std::string kLanguageCode = "en-US";
-static const std::string kDeviceInstanceId = "default";
 static const std::string kDeviceModelId = "default";
+static const std::string kDeviceInstanceId = "default";
 
 bool verbose = false;
 
@@ -80,24 +82,26 @@ void PrintUsage() {
             << "--credentials <credentials_file> "
             << "[--api_endpoint <API endpoint>] "
             << "[--locale <locale>]"
+            << "[--html_out <command to load HTML file>]"
             << std::endl;
 }
 
 bool GetCommandLineFlags(
     int argc, char** argv, std::string* credentials_file_path,
-    std::string* api_endpoint, std::string* locale) {
+    std::string* api_endpoint, std::string* locale, std::string* html_out_command) {
   const struct option long_options[] = {
     {"credentials", required_argument, nullptr, 'c'},
     {"api_endpoint",     required_argument, nullptr, 'e'},
     {"locale",           required_argument, nullptr, 'l'},
     {"verbose",          no_argument, nullptr, 'v'},
+    {"html_out",         required_argument, nullptr, 'h'},
     {nullptr, 0, nullptr, 0}
   };
   *api_endpoint = ASSISTANT_ENDPOINT;
   while (true) {
     int option_index;
     int option_char =
-        getopt_long(argc, argv, "c:e:l:v", long_options, &option_index);
+        getopt_long(argc, argv, "c:e:l:v:h", long_options, &option_index);
     if (option_char == -1) {
       break;
     }
@@ -114,6 +118,9 @@ bool GetCommandLineFlags(
       case 'v':
         verbose = true;
         break;
+      case 'h':
+        *html_out_command = optarg;
+        break;
       default:
         PrintUsage();
         return false;
@@ -123,12 +130,12 @@ bool GetCommandLineFlags(
 }
 
 int main(int argc, char** argv) {
-  std::string credentials_file_path, api_endpoint, locale, text_input_source;
+  std::string credentials_file_path, api_endpoint, locale, text_input_source, html_out_command;
   // Initialize gRPC and DNS resolvers
   // https://github.com/grpc/grpc/issues/11366#issuecomment-328595941
   grpc_init();
   if (!GetCommandLineFlags(argc, argv, &credentials_file_path,
-                          &api_endpoint, &locale)) {
+                          &api_endpoint, &locale, &html_out_command)) {
     return -1;
   }
 
@@ -156,6 +163,11 @@ int main(int argc, char** argv) {
     assist_config->mutable_audio_out_config()->set_sample_rate_hertz(16000);
     assist_config->set_text_query(text_input_source);
 
+    // Set parameters for screen config
+    assist_config->mutable_screen_out_config()->set_screen_mode(
+      html_out_command.empty() ? ScreenOutConfig::SCREEN_MODE_UNSPECIFIED : ScreenOutConfig::PLAYING
+    );
+
     // Read credentials file.
     std::ifstream credentials_file(credentials_file_path);
     if (!credentials_file) {
@@ -210,11 +222,16 @@ int main(int argc, char** argv) {
                     << ")" << std::endl;
         }
       }
-      if (response.dialog_state_out().supplemental_display_text().size() > 0) {
-        // CUSTOMIZE: render spoken response on screen
-        std::clog << "assistant_sdk response:" << std::endl;
-        std::cout << response.dialog_state_out().supplemental_display_text()
-                  << std::endl;
+      if (!html_out_command.empty() && response.screen_out().data().size() > 0) {
+        std::string html_out_base64 = base64_encode(response.screen_out().data());
+        system((html_out_command + " \"data:text/html;base64, " + html_out_base64 + "\"").c_str());
+      } else if (html_out_command.empty()) {
+        if (response.dialog_state_out().supplemental_display_text().size() > 0) {
+          // CUSTOMIZE: render spoken response on screen
+          std::clog << "assistant_sdk response:" << std::endl;
+          std::cout << response.dialog_state_out().supplemental_display_text()
+                    << std::endl;
+        }
       }
     }