Dialogflow fulfillment not showing suggestions on Google Assistant - dialogflow-es

I've written my own library to respond to a Dialog Fulfillment webhook, but it's not working as the docs say that it should.
When my webhook is used for fulfilment the simple text response works, but suggestion chips do not. The "RESPONSE" tab in the Dialogflow console shows the response from my webhook:
{
"outputContexts": [ ],
"fulfillmentText": "Hi, how can I help you?",
"fulfillmentMessages": [
{ "text": {"text": ["Hi, how can I help you?"]} },
{ "suggestions": {"suggestions": [{"title": "Looking for a job"}]} }
],
"payload": {
"google": {
"expectUserResponse": true,
"expectedInputs": [{
"possibleIntents": [{"intent": "actions.intent.TEXT"}, {"intent": "JobSearch"}],
"speechBiasingHints": ["looking for a job"],
"inputPrompt": {
"richInitialPrompt": {
"items": [
{"simpleResponse": { "textToSpeech": "Hi, How can I help you?"}}
],
"suggestions": [{"title": "Looking for a job"}]
}
}
}
]
}
}
}
When I disable my webhook and and provide a response message and suggestions in the console the suggestions do show, but the RESPONSE tab looks more like my payload.google field:
{
"conversationToken": "[]",
"expectUserResponse": true,
"expectedInputs": [
{
"inputPrompt": {
"richInitialPrompt": {
"items": [
{"simpleResponse": {"textToSpeech": "Response from Dialogflow"}}
],
"suggestions": [{"title": "Dialogflow suggestion"}]
}
},
"possibleIntents": [{"intent": "assistant.intent.action.TEXT"}],
"speechBiasingHints": []
}
],
"responseMetadata": {
"status": {"message": "Success (200)"},
"queryMatchInfo": {"queryMatched": true, "intent": "..."}
}
}

Oh - the payload in the Dialogflow webhook response does NOT have the same format as the Conversation webhook response.
They have similarities, but also important differences.

Related

Malformed Response: Failed to parse Dialogflow response into AppResponse because of empty speech response

I am using firebase function for the webhook fulfillment in Dialogflow. I am getting webhook successful as a fulfillment status but it is not working. I am using version 1. When I test it on Google Assistant simulator, it says "App is not responding".
firebase function
const functions = require('firebase-functions');
exports.webhook = functions.https.onRequest((request, response) => {
response.send({
"google":{
"richResponse":{
"items":[
{
"simpleResponse":{
"textToSpeech":"Hey! Good to see you."
}
},
{
"mediaResponse":{
"mediaType":"AUDIO",
"mediaObjects":[
{
"name":"Exercises",
"description":"ex",
"largeImage":{
"url":"http://res.freestockphotos.biz/pictures/17/17903-balloons-pv.jpg",
"accessibilityText":"..."
},
"contentUrl":"https://theislam360.me:8080/hbd.mp3"
}
]
}
}
],
"suggestions":[
{
"title":"chips"
}
]
}
}
}
)
});`
When I copy paste the response from {google... to the end in the custom payload manually via GUI, It works. While for webhook, it is not working.
RAW API RESPONSE
{
"id": "eaf627ed-26b5-4965-b0b0-bc77144e144b",
"timestamp": "2019-04-15T11:54:18.948Z",
"lang": "en",
"result": {
"source": "agent",
"resolvedQuery": "play hbd",
"action": "",
"actionIncomplete": false,
"parameters": {
"any": "hbd"
},
"contexts": [],
"metadata": {
"isFallbackIntent": "false",
"webhookResponseTime": 34,
"intentName": "play",
"intentId": "e60071cd-ce31-4ef9-ae9b-cc370c3362b3",
"webhookUsed": "true",
"webhookForSlotFillingUsed": "false"
},
"fulfillment": {
"messages": []
},
"score": 1
},
"status": {
"code": 200,
"errorType": "success"
},
"sessionId": "e91bd62f-766b-b19d-d37b-2917ac20caa6"
}
FULFILLMENT REQUEST
{
"id": "eaf627ed-26b5-4965-b0b0-bc77144e144b",
"timestamp": "2019-04-15T11:54:18.948Z",
"lang": "en",
"result": {
"source": "agent",
"resolvedQuery": "play hbd",
"speech": "",
"action": "",
"actionIncomplete": false,
"parameters": {
"any": "hbd"
},
"contexts": [],
"metadata": {
"intentId": "e60071cd-ce31-4ef9-ae9b-cc370c3362b3",
"webhookUsed": "true",
"webhookForSlotFillingUsed": "false",
"isFallbackIntent": "false",
"intentName": "play"
},
"fulfillment": {
"speech": "",
"messages": []
},
"score": 1
},
"status": {
"code": 200,
"errorType": "success"
},
"sessionId": "e91bd62f-766b-b19d-d37b-2917ac20caa6"
}
FULFILLMENT RESPONSE
{
"google": {
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "Hey! Good to see you."
}
},
{
"mediaResponse": {
"mediaType": "AUDIO",
"mediaObjects": [
{
"name": "Exercises",
"description": "ex",
"largeImage": {
"url": "http://res.freestockphotos.biz/pictures/17/17903-balloons-pv.jpg",
"accessibilityText": "..."
},
"contentUrl": "https://theislam360.me:8080/hbd.mp3"
}
]
}
}
],
"suggestions": [
{
"title": "chips"
}
]
}
}
}
FULFILLMENT STATUS
Webhook execution successful
Firebase Logs
Google Assistant Simulator Logs
You're not using the correct JSON in the response. By putting it in the GUI in the "custom payload" section, it is creating a larger JSON response for you. The google object needs to be under the data object for Dialogflow v1 or payload for Dialogflow v2. (And if you haven't switched to v2 - you should do so immediately, since v1 will be switched off in about a month.)
So what you're returning should look more like
{
"payload": {
"google": {
...
}
}
}

How to store data in conversation using Dialogflow webhook

I am using dialogflow webhook and want to store data in conversation.
Below is what i understood from dialogflow docs
response.setHeader('Content-Type', 'application/json');
response.send(JSON.stringify({
"payload": {
"google": {
"expectUserResponse": true,
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "OKAY"
}
}
]
},
"conversationToken" : "count=1"
}
}
}))
This does not work as the JSON from the next request does not have this stored value.
You can check my complete answer here.
In short use contexts to store parameters.
Use the output context to save parameters
{
"fulfillmentText":"This is a text response",
"fulfillmentMessages":[ ],
"source":"example.com",
"payload":{
"google":{ },
"facebook":{ },
"slack":{ }
},
"outputContexts":[
{
"name":"<Context Name>",
"lifespanCount":5,
"parameters":{
"<param name>":"<param value>"
}
}
],
"followupEventInput":{ }
}

Dialogflow change surface from webhook

I am trying to send a response from my webhook to trigger the process to change to another surface, but Actions on Google always throws:
MalformedResponse 'final_response' must be set.
And that it's not very helpful.
This is the JSON I am returning:
{
"payload": {
"google": {
"expectUserResponse": true,
"conversationToken": "{\"data\":{}}",
"userStorage": "{\"data\":{}}",
"expectedInputs": [
{
"inputPrompt": {
"richInitialPrompt": {
"items": [
{
"simpleResponse": {
"textToSpeech": "PLACEHOLDER"
}
}
]
}
},
"possibleIntents": [
{
"intent": "actions.intent.NEW_SURFACE",
"inputValueData": {
"#type": "type.googleapis.com/google.actions.v2.NewSurfaceValueSpec",
"capabilities": [
{
"name": "actions.capability.SCREEN_OUTPUT"
}
],
"context": "Sure, I have some sample images for you.",
"notificationTitle": "Sample Images"
}
}
]
}
]
}
}
}
On Dialogflow I have setup 2 intents; one intent that returns the json specified here, and another intent with the event actions_intent_NEW_SURFACE, so I know what the user responded to the question to change surface.
I have been reading these sites:
https://developers.google.com/actions/assistant/surface-capabilities#multi-surface_conversations
https://dialogflow.com/docs/reference/api-v2/rest/Shared.Types/WebhookResponse
https://developers.google.com/actions/build/json/dialogflow-webhook-json#dialogflow-response-body
Action On Google, webhook response with actions.intent.NEW_SURFACE (This seems like the same problem has I have, but the OP didn't write the response.)
But none of them throw me some light on this matter.
It looks like you're trying to send a full Action SDK body as part of your Dialogflow response. While the payload.google portion of the JSON contains objects that are similar to the Action SDK response, there are some differences. You can see https://developers.google.com/actions/build/json/dialogflow-webhook-json#dialogflow-response-body for a full example of what the response should be when using a helper, but I think your response needs to be something like
{
"payload": {
"google": {
"expectUserResponse": true,
"userStorage": "{\"data\":{}}",
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "PLACEHOLDER"
}
}
]
},
"systemIntent": {
"intent": "actions.intent.NEW_SURFACE",
"inputValueData": {
"#type": "type.googleapis.com/google.actions.v2.NewSurfaceValueSpec",
"capabilities": [
"actions.capability.SCREEN_OUTPUT"
],
"context": "Sure, I have some sample images for you.",
"notificationTitle": "Sample Images"
}
}
}
}
}

dialogflow webhook not considering payload using fulfillmentText, fulfillmentMessages

FULFILLMENT REQUEST
{
"responseId": "4955f972-058c-44c2-a9c6-fe2c1d846fcd",
"queryResult": {
"queryText": "dsnaf",
"action": "intentNotMatched",
"parameters": {},
"allRequiredParamsPresent": true,
"fulfillmentText": "I think I may have misunderstood your last statement.",
"fulfillmentMessages": [
{
"text": {
"text": [
"I'm afraid I don't understand."
]
}
}
],
"outputContexts": [
{
"name": "****",
"lifespanCount": 1
}
],
"intent": {
"name": "****",
"displayName": "Default Fallback Intent",
"isFallback": true
},
"intentDetectionConfidence": 1,
"languageCode": "en"
},
"originalDetectIntentRequest": {
"payload": {}
},
"session": "****"
}
FULFILLMENT RESPONSE
{
"payload": {
"google": {
"expectUserResponse": true,
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "I'm sorry. I didn't quite grasp what you just said."
}
}
]
},
"userStorage": "{\"data\":{}}"
}
},
"outputContexts": [
{
"name": "***",
"lifespanCount": 99,
"parameters": {
"data": "{}"
}
}
]
}
RAW API RESPONSE
{
"responseId": "4955f972-058c-44c2-a9c6-fe2c1d846fcd",
"queryResult": {
"queryText": "dsnaf",
"action": "intentNotMatched",
"parameters": {},
"allRequiredParamsPresent": true,
"fulfillmentMessages": [
{
"text": {
"text": [
"I'm afraid I don't understand."
]
}
}
],
"webhookPayload": {
"google": {
"userStorage": "{\"data\":{}}",
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "I'm sorry. I didn't quite grasp what you just said."
}
}
]
},
"expectUserResponse": true
}
},
"outputContexts": [
{
"name": "*****",
"lifespanCount": 99,
"parameters": {
"data": "{}"
}
},
{
"name": "******",
"lifespanCount": 1
}
],
"intent": {
"name": "****",
"displayName": "Default Fallback Intent",
"isFallback": true
},
"intentDetectionConfidence": 1,
"diagnosticInfo": {
"webhook_latency_ms": 286
},
"languageCode": "en"
},
"webhookStatus": {
"message": "Webhook execution successful"
}
}
Google Assistance response
USER SAYS dsnaf
DEFAULT RESPONSE I'm afraid I don't understand.
CONTEXTS
_actions_on_google,initial_chat
INTENT Default Fallback Intent
IN Google Assistance response is default fulfillmentText instead payload google richresponse
Where are you testing this? If you're in the test console, it's always going to show you the simple text response. You'll need to specifically select Google Assistant in the test console to see the rich response for that platform:

Dialogflow v2 API - cards not shown in the simulator

I've a webhook for fulfillment.
Below is the code that's responding back
let result_obj = {
"fulfillmentText": "This is a text response",
"fulfillmentMessages": [
{
"text": {
"text": [
"this is test"
]
}
},
{
"card": {
"title": "card title",
"subtitle": "card text",
"imageUri": "https://assistant.google.com/static/images/molecule/Molecule-Formation-stop.png",
"buttons": [
{
"text": "button text",
"postback": "https://assistant.google.com/"
}
]
}
}
]
}
Below is the result from dialogflow GUI
Below is what I get when I run from the simulator or from the Google Assistant application on the Android phone
Both the simulator and phone are not showing the cards. Am I missing something obvious here?
For rich responses like cards to show on Google Assistant you have to use the payload part of response JSON, here is an example:
{
"fulfillmentText": "This is a text response",
"fulfillmentMessages": [],
"source": "example.com",
"payload": {
"google": {
"expectUserResponse": true,
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "This is a Basic Card:"
}
},
{
"basicCard": {
"title": "card title",
"image": {
"url": "https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png",
"accessibilityText": "Google Logo"
},
"buttons": [
{
"title": "Button Title",
"openUrlAction": {
"url": "https://www.google.com"
}
}
],
"imageDisplayOptions": "WHITE"
}
}
]
}
}
},
"outputContexts": [],
"followupEventInput": {}
}
Check out this github repo for all rich-responses' JSON formats.

Resources