I am using Dialogflow and would like to implement a media response in my project. I have tried to do so using the following link: https://developers.google.com/actions/assistant/responses but have been unsuccessful. How to do the implementation?
const functions = require('firebase-functions')
const {dialogflow} = require('actions-on-google')
const TEST = 'test'
const app = dialogflow({
debug: true,
})
app.intent('test', (conv) =>{
conv.ask('we will now play a song for you');
conv.ask(new MediaObject({
name: 'Jazz in Paris',
url: 'https://storage.googleapis.com/automotive-media/Jazz_In_Paris.mp3',
}));
conv.ask(new MediaResponse({
mediaObject: 'Jazz in Paris',
mediaType: AUDIO,
}));
});
exports.dialogflowFirebaseFulfillment = functions.https.onRequest(app)
You just need to add the MediaObject. You don't need to add the MediaResponse object since the library will add it for you.
You will, however, need to make sure you load the MediaObject object as part of your call to require(). You'll also need to load the Suggestions object since you need to provide suggestions to advance the conversation if the user chooses to interrupt your audio.
So your code can look something like this:
const functions = require('firebase-functions')
const {
dialogflow,
MediaObject,
Suggestions
} = require('actions-on-google')
const app = dialogflow({
debug: true,
})
app.intent('test', (conv) =>{
conv.ask('we will now play a song for you');
conv.ask(new MediaObject({
name: 'Jazz in Paris',
url: 'https://storage.googleapis.com/automotive-media/Jazz_In_Paris.mp3',
}));
conv.ask(new Suggestions(['suggestion 1', 'suggestion 2']));
});
exports.dialogflowFirebaseFulfillment = functions.https.onRequest(app)
Related
Has anyone experienced this issue?
I am sure the dialogflow connection works because listing the intents works which is (intentsClient.listIntents(request)) and it gives me back all the intents from my dialogflow es agent. So it looks like the credentials should be fine.
Any help would be highly appreciated,
Thanks
const dialogflow = require('#google-cloud/dialogflow').v2;
const uuid = require('uuid');
const sessionId = uuid.v4();
const sessionClient = new dialogflow.SessionsClient({ keyFilename: '..........' });
const sessionPath = sessionClient.projectAgentSessionPath(
projectId,
sessionId
);
const query = 'What are the pets rules';
async function detectIntentForQuestion() {
// The text query request.
const request = {
session: sessionPath,
queryInput: {
text: {
text: query,
languageCode: 'en-US',
},
},
queryParams: {
sentimentAnalysisRequestConfig: {
analyzeQueryTextSentiment: true,
},
},
};
// Send request and log result
const responses = await sessionClient.detectIntent(request);
console.log('Detected intent');
const result = responses[0].queryResult;
console.log(` Query: ${result.queryText}`);
console.log(` Response: ${result.fulfillmentText}`);
if (result.intent) {
console.log(` Intent: ${result.intent.displayName}`);
} else {
console.log(' No intent matched.');
}
if (result.sentimentAnalysisResult) {
console.log('Detected sentiment');
console.log(
` Score: ${result.sentimentAnalysisResult.queryTextSentiment.score}`
);
console.log(
` Magnitude: ${result.sentimentAnalysisResult.queryTextSentiment.magnitude}`
);
} else {
console.log('No sentiment Analysis Found');
}
};
you are setting languageCode: 'en-US' for language. Does your agent support this language? If it is set using a different language (ie. Spanish), then it is expected to give you back only the fallback intent response.
other thing that you can do is checking at https://dialogflow.cloud.google.com/#/agent/<your agent name>/history and in your last interactions, you click on the 3 dots in the agent response and go to Raw interaction log. There you can see the information the agent got from your code, how it was interpreted by the agent and how it answered you back.
I am using botframework v4, but coming over from v3, I have not found any documentation that is similar to the code I use below but for v4, regarding sending proactive messages from Azure Function App
Below is the code I previously used but am having trouble adapting:
var builder = require('botbuilder');
// setup bot credentials
var connector = new builder.ChatConnector({
appId: process.env.MICROSOFT_APP_ID,
appPassword: process.env.MICROSOFT_APP_PASSWORD
});
module.exports = function (context, req) {
if (req.body) {
var savedAddress = req.body.channelAddress;
var inMemoryStorage = new builder.MemoryBotStorage();
var bot = new builder.UniversalBot(connector).set('storage', inMemoryStorage);
sendProactiveMessage(savedAddress, bot)
}
};
function sendProactiveMessage(address, bot) {
var msg = new builder.Message().address(address);
msg.textLocale('en-US');
var img = {
attachments: [{
contentType: "image/jpg",
contentUrl: latestUrl,
}]
};
msg.addAttachment(img.attachments[0]);
msg.text('hello');
bot.send(msg);
}
This works fine with v3 but not v4.
If possible I would also like to find a way to log a user out:
await botAdapter.signOutUser(innerDc.context, this.connectionName);
This is how I do it in the bot itself, but doing so from Azure Functions again is proving difficult.
Any help would be appreciated.
Great that you are making the move from v3 to v4! Have you had a look at Send proactive notifications to users? This example is pretty straight forward and can be used within an Azure function.
First you retrieve the Conversation Reference in your bot by calling TurnContext.getConversationReference(context.activity);. This is the reference you could use in your proactive function to open the conversation. In your case you provide that via the request body to a proactive function, so I will do the same in my example.
My proactive endpoint example is written in Typescript, however it works the same way in plain Javascript. Create a HTTP trigger in Azure Functions and use the following code. I have added comments inline for clarity.
const { BotFrameworkAdapter } = require('botbuilder');
// Create adapter.
// If you need to share this adapter in multiple functions, you could
// instantiate it somewhere else and import it in every function.
const adapter = new BotFrameworkAdapter({
appId: process.env.MicrosoftAppId,
appPassword: process.env.MicrosoftAppPassword
});
module.exports = async function (context, req) {
// Validate if request has a body
if (!req.body) {
context.res = {
status: 400,
body: "Please pass a conversation reference in the request body"
};
return;
}
// Retrieve conversation reference from POST body
const conversationReference = req.body;
// Open the conversation and retrieve a TurnContext
await adapter.continueConversation(conversationReference, async turnContext => {
// Send a text activity
// Depending on the channel, you might need to use https://aka.ms/BotTrustServiceUrl
await turnContext.sendActivity('Proactive hello');
});
context.res = {
body: 'Message sent!'
};
};
In the end you could make a request to this Azure Function, where you pass the Conversation Reference as body of the type application/json.
Extending this example with features like signOutUser is simple. You can call all functions within the continueConversation function, just as in a normal bot. You can even receive the adapter object there if you wish.
await adapter.continueConversation(conversationReference, async turnContext => {
// Sign user out
turnContext.adapter.signOutUser(turnContext, 'connection-name');
});
Hi i have created a dialogflow nodejs backend which detects intents using the client library for nodejs.
const sessionPath = this.sessionClient.sessionPath(this.configService.get('dialogFlowProjectId'), sessionId);
const request = {
session: sessionPath,
queryInput: {
text: {
text: query,
languageCode: "en-US"
}
}
};
// Send request and log result
Logger.log(request);
const responses = await this.sessionClient.detectIntent(request);
this works fine but I also want to trigger a fulfillment for certain intents.
I have setup a webhook url - this works fine when you use the chat in the dialogflow console. But when I use the method that I have created to send the request to dialogflow the webhook doesn't get called and goes to fallback intent. I'm calling the same intent through the dialogflow console chat and through my own API.
How can I trigger the webhook call when I use the client library API?
First, remember that you don't "call an Intent", either through the test console or through the API. What you can do is send query text that should be matched by an Intent.
If you have set the Intent so that it should call the Fulfillment webhook, then this should happen no matter the source, as long as the Intent itself is triggered.
That the Fallback Intent is getting triggered instead suggests that something about the query text isn't matching the Intent you think it should be matching. I would check to make sure the text you're sending in query should match the training phrases for the Intent in question, that you have no Input Contexts, and that your agent is set for the "en-US" language code.
For routing multiple intents through a webhook you're going to want to have a handler.js file with an express router in it...
const def = require('./intents/default')
const compression = require('compression')
const serverless = require('serverless-http')
const bodyParser = require('body-parser')
const { WebhookClient } = require('dialogflow-fulfillment')
const express = require('express')
const app = express()
// register middleware
app.use(bodyParser.json({ strict: false }))
app.use(function (err, req, res, next) {
res.status(500)
res.send(err)
})
app.use(compression())
app.post('/', async function (req, res) {
// Instantiate agent
const agent = new WebhookClient({ request: req, response: res })
const intentMap = new Map()
intentMap.set('Default Welcome Intent', def.defaultWelcome)
await agent.handleRequest(intentMap)
})
module.exports.server = serverless(app)
As you can see, this handler.js is using serverless-http and express.
Your intent.js function could look something like this ...
module.exports.defaultWelcome = async function DefaultWelcome (agent) {
const responses = [
'Hi! How are you doing?',
'Hello! How can I help you?',
'Good day! What can I do for you today?',
'Hello, welcoming to the Dining Bot. Would you like to know what\'s being served? Example: \'What\'s for lunch at cronkhite?\'',
'Greetings from Dining Bot, what can I do for you?(Hint, you can ask things like: \'What\'s for breakfast at Annenberg etc..\' or \'What time is breakfast at Anennberg?\')',
'greetings',
'hey',
'long time no see',
'hello',
"lovely day isn't it"
]
const index = Math.floor((Math.random() * responses.length) + 1)
console.log(responses[index].text)
agent.add(responses[index].text)
}
This gives you the bare bones for routing multiple request through your webhook. Hope this helps.
I was making dialogflow app that links the user to each links regarding to the number that user says.
I got google codelab's fulfillment code and edited it to make it.
But when I try to use Card and other things to create clickable url links, "MalformedResponse: Failed to parse Dialogflow response into AppResponse because of empty speech response" happens.
I'm using Dialogflow InlineEditor for this.
Tried to app.buildRichResponse but that also fails
This is my node.js code.
'use strict';
// Import the Dialogflow module from the Actions on Google client library.
const {dialogflow} = require('actions-on-google');
// Import the firebase-functions package for deployment.
const functions = require('firebase-functions');
// Instantiate the Dialogflow client.
const app = dialogflow({debug: true});
const {WebhookClient} = require('dialogflow-fulfillment');
const {Card, Suggestion} = require('dialogflow-fulfillment');
// Handle the Dialogflow intent named 'favorite color'.
// The intent collects a parameter named 'color'.
app.intent('getJobs', (conv, {number}) => {
const jobNum = number;
// Respond with the user's lucky number and end the conversation.
if (number == 1) {
conv.ask(new Card({
title: `Title: this is a card title`,
imageUrl: 'https://dialogflow.com/images/api_home_laptop.svg',
text: `This is the body text of a card. You can even use line\n
breaks and emoji! 💁`,
buttonText: 'This is a button',
buttonUrl: 'https://docs.dialogflow.com/'
}));
} else if (number == 2) {
conv.close('camera on');
} else {
conv.close('unknown number');
}
});
//Set the DialogflowApp object to handle the HTTPS POST request.
exports.dialogflowFirebaseFulfillment = functions.https.onRequest(app);
I think problem is, you use only rich response when number is 1 and this may cause "MalformedResponse: Failed to parse Dialogflow response into AppResponse because of empty speech response" error since there is nothing assistant can response with voice. You can't use rich response without simple response.
Also you are trying to use both actions-on-google and dialogflow-fulfillmentlibraries. This can cause the problem as well. You can try:
function getJobs(agent) {
agent.add(`This message is from Dialogflow's Cloud Functions for Firebase editor!`);
agent.add(new Card({
title: `Title: this is a card title`,
imageUrl: 'https://developers.google.com/actions/images/badges/XPM_BADGING_GoogleAssistant_VER.png',
text: `This is the body text of a card. You can even use line\n breaks and emoji! 💁`,
buttonText: 'This is a button',
buttonUrl: 'https://assistant.google.com/'
})
);
intentMap.set('getJobs', getJobs);
or
const { dialogflow, BasicCard, Image, Button } = require('actions-on-google');
app.intent('getJobs', (conv, {number}) => {
conv.ask('This is simple response...',new BasicCard({
text: `This is a basic card.`
subtitle: 'This is a subtitle',
title: 'Title: this is a title',
buttons: new Button({
title: 'This is a button',
url: 'https://assistant.google.com/',
}),
image: new Image({
url: 'https://example.com/image.png',
alt: 'Image alternate text',
}),
display: 'CROPPED',
}));
});
I'm trying to display a BasicCard in dialogflow's fulfillment, but I always get this error message MalformedResponse 'final_response' must be set.. The responseMetaData says code 10 and Failed to parse Dialogflow response into AppResponse because of empty speech response.
This is my code:
agent.add(`Check this out`);
agent.add(new BasicCard({
title: 'Card Title',
text: 'Description',
image: {
url: 'https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png',
accessibilityText: 'Google Logo',
},
buttons: new Button({
title: 'Button Title',
url: 'https://www.google.com',
}),
}));
I'm using Cloud Functions for Firebase and
"actions-on-google": "2.0.0-alpha.4",
"dialogflow-fulfillment": "0.3.0-beta.3"
and
const {WebhookClient} = require('dialogflow-fulfillment');
const {Card} = require('dialogflow-fulfillment');
const {BasicCard, Button} = require('actions-on-google');
const agent = new WebhookClient({ request, response });
It works perfectly fine when I use new Card() instead of new BasicCard().
Any ideas what I'm doing wrong?
Just a thought, have you tried
let conv=agent.conv();
conv.ask(new BasicCard({//Your code here
});