I have developed a Microsoft Teams App Bot. Im running it in Firebase as a function.
This is the function:
exports.teamsBot = functions
.region("europe-west1")
.https.onRequest(async (req, res) => {
await adapter.process(req, res,
(context) => bot.run(context)
);
});
adapter is a CloudAdapter.
I have a card with 3 actions:
{
"type": "ActionSet",
"actions": [
{
"type": "Action.Submit",
"title": "π",
"data": {
"action": "feedback-up",
"feedbackId": feedback.id
}
},
{
"type": "Action.Submit",
"title": "π",
"data": {
"action": "feedback-down",
"feedbackId": feedback.id
}
},
{
"type": "Action.Submit",
"title": "π Did not attend",
"data": {
"action": "feedback-no-show",
"feedbackId": feedback.id
}
}
]
}
I retrieve the result in onMessage in my bot. I do some magic and store the values and reply with:
await context.sendActivity("Thanks for your valuable feedback... π");
The process works however I get a strange annoying issue.
First when I click the button I direct get: "Unable to reach app. Please try again." and then like 1 second later when my reply hits it changes to I guess the correct one "Your response was sent to the app".
So from a user perspective it looks like something is first wrong then its correct. It feels like the Action is waiting for some sort of first reply that Im missing. Then when my reply comes it feels like ouuu its okay now.
Anybody has any Idea on this?
I am building a bot so whenever a specific task is initiated I want to send an image to the user.
exports.handler = function(context, event, callback) {
let response = {
"actions": [
{
"show": {
"body": "Twilio Owls",
"images": [{
"label": "Original Owl",
"url": "https://demo.twilio.com/owl.png"
}]
}
}]
}
callback(null, response)
}
I tried using the above code inside my function and I have linked it to my Twilio autopilot task, It is working in the simulator, but when I am testing it on Whatsapp Image does not appear in the Whatsapp chat only body get displayed.
I tested and received the owl image. I noticed have a rogue {, maybe that is it?
exports.handler = function(context, event, callback) {
let response = {
"actions": [
{
"show": {
"body": "Twilio Owls",
"images": [{
"label": "Original Owl",
"url": "https://demo.twilio.com/owl.png"
}]
}
}]
}
callback(null, response)
};
I have an Entity that is supposed to be updated on a per-session basis with user-specific information. This had worked when I was using Dialogflow v1, and I thought it had worked with v2, but I'm now having significant problems with it.
I believe I am setting the Session Entity information correctly, but for the Intent that uses it, it only matches when a value from the Developer Entity is used.
How can I get it to use the Session Entity? Am I doing something wrong when updating it? Am I using the wrong Session ID? Is there a way I can better verify or test that I'm using the correct ID or that I'm updating the Entity correctly? Is this just a bug?
Documentation of everything follows.
The project is configured to use v2 and to allow for beta features, although I've tried this without the beta features as well.
There are only three Intents. A Fallback Intent to capture failures, a Welcome Intent that gets the welcome event, and the "entry" intent that is supposed to capture the entry code which should match the "code" Entity. All of them use a webhook for fulfillment.
The Fallback Intent
The Welcome Intent
The "entry" Intent
As shown in the "entry" Intent, it uses the "code" Entity, which is the only Developer Entity in the system
The code has most of the Dialogflow specific work in a separate module that uses the "dialogflow" module from npm to set the Session Entity. (Note this is different than the dialogflow-fulfillment module, which is used to handle fulfillment. I'm using the multivocal library for fulfillment, but that shouldn't matter.) (It also uses firebase functions to run on, but I don't think any of these are relevant.)
From the package.json:
"dependencies": {
"dialogflow": "^0.9.0",
"firebase-admin": "~7.0.0",
"firebase-functions": "^2.2.0",
"multivocal": "^0.11.1"
},
This is imported as dialogflow, specifying the API version to use:
const dialogflow = require('dialogflow').v2beta1;
The functions I show below call envToConfig(env) which takes the environment (a multivocal concept that just stores relevant information, including the Dialogflow parent and certificate information) and returns the configuration that needs to be passed to dialogflow.SessionEntityTypesClient( config ). Given no errors are thrown in further calls, it appears to work correctly.
The makeEntityType( name, entityMap ) function takes a map of values to be used for the entities in a SessionEntityType and returns an object that will be used to build a full SessionEntityType. The name provided here is the display name.
function makeEntityType( name, entityMap ){
let ret = {
displayName: name,
entities: []
};
Object.keys( entityMap ).map( key => {
let val = entityMap[key];
let entity = {
value: key,
synonyms: [key, ...val]
};
ret.entities.push( entity );
});
return ret;
}
exports.makeEntityType = makeEntityType;
The result from this is passed to setSessionEntity( env, entityType ) along with the multivocal environment, which contains some information we use in the session. It makes sure the name and entityOverrideMode are set correctly in the entityType and then tries to create it. I've tried using PATCH as well, and it behaves the same way. It also does a bunch of logging, that I'll show later when it runs to prove it actually works.
function setSessionEntity( env, entityType ){
const config = envToConfig( env );
const client = new dialogflow.SessionEntityTypesClient( config );
let parent = env.dialogflow.parent;
if( entityType.displayName && !entityType.name ){
entityType.name = `${parent}/entityTypes/${entityType.displayName}`;
}
if( !entityType.entityOverrideMode ){
entityType.entityOverrideMode = 'ENTITY_OVERRIDE_MODE_OVERRIDE';
}
console.log('setSessionEntity parent',parent);
const request = {
parent: parent,
sessionEntityType: entityType
};
console.log('setSessionEntity request',JSON.stringify(request,null,1));
return client.createSessionEntityType( request )
.then( create => {
console.log('setSessionEntity created',JSON.stringify(create,null,1));
return Promise.resolve( env );
})
.catch( err => {
console.error('setSessionEntity problem creating',err);
return Promise.resolve( env );
})
}
exports.setSessionEntity = setSessionEntity;
For debugging, I also have a function that lists the session entities:
function listSessionEntities( env ){
let parent = env.dialogflow && env.dialogflow.parent;
console.log('listSessionEntities parent', parent);
if( !parent ){
return Promise.resolve( env );
}
const config = envToConfig( env );
const client = new dialogflow.SessionEntityTypesClient( config );
const request = {
parent: parent
};
return client.listSessionEntityTypes(request)
.then( result => {
console.log('listSessionEntities', JSON.stringify(result,null,1));
})
.catch( err => {
console.log('listSessionEntities err', err);
})
.then( () => Promise.resolve( env ) );
}
exports.listSessionEntities = listSessionEntities;
The code that calls this imports it as Dialogflow:
const Dialogflow = require('./dialogflow');
As part of all the webhook calls, the listSessionEntities() function is called before any specific handler is:
function debugSessionEntities( env ){
return Dialogflow.listSessionEntities( env );
}
When the Welcome Intent is triggered, it sets the "code" Session Entity to have two new types that should override the type that was defined in the "code" Developer Entity:
function handleWelcome( env ){
const entityType = Dialogflow.makeEntityType('code',{
'alpha': [],
'bravo': []
});
return Dialogflow.setSessionEntity( env, entityType )
.then( env => Multivocal.handleDefault( env ) );
}
When I run this through the simulator, it doesn't pick up on the Session Entity Types that are set, but do still respond to the Developer Entity Type. (Using a real device works the same way.)
In the simulator, this is what it reports in the Request tab for the Welcome Intent:
{
"responseId": "55a9eb06-ce05-48f9-8a56-b993fa512aee",
"queryResult": {
"queryText": "GOOGLE_ASSISTANT_WELCOME",
"action": "multivocal.welcome",
"parameters": {},
"allRequiredParamsPresent": true,
"fulfillmentText": "Hello! How can I help you?",
"fulfillmentMessages": [
{
"text": {
"text": [
"Greetings! How can I assist?"
]
}
}
],
"outputContexts": [
{
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/contexts/google_assistant_welcome"
},
{
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/contexts/actions_capability_screen_output"
},
{
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/contexts/actions_capability_audio_output"
},
{
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/contexts/actions_capability_account_linking"
},
{
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/contexts/google_assistant_input_type_keyboard"
},
{
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/contexts/actions_capability_media_response_audio"
},
{
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/contexts/actions_capability_web_browser"
}
],
"intent": {
"name": "projects/session-test-XXXXX/agent/intents/ca79c951-4d75-4b2b-acd4-7dac2f81856e",
"displayName": "welcome"
},
"intentDetectionConfidence": 1,
"languageCode": "en-us"
},
"originalDetectIntentRequest": {
"source": "google",
"version": "2",
"payload": {
"isInSandbox": true,
"surface": {
"capabilities": [
{
"name": "actions.capability.SCREEN_OUTPUT"
},
{
"name": "actions.capability.ACCOUNT_LINKING"
},
{
"name": "actions.capability.AUDIO_OUTPUT"
},
{
"name": "actions.capability.MEDIA_RESPONSE_AUDIO"
},
{
"name": "actions.capability.WEB_BROWSER"
}
]
},
"requestType": "SIMULATOR",
"inputs": [
{
"rawInputs": [
{
"query": "Talk to my test app",
"inputType": "KEYBOARD"
}
],
"intent": "actions.intent.MAIN"
}
],
"user": {
"userStorage": "{\"UserId\":\"ABwppHHd40lIZ1o0bRERAKlHNtNcS2qFtz7NbRQnb31AQDFuV41VPFQivXwwpQGtv_5SlsZNp0N3kxalIIXXXXXX\",\"NumVisits\":1}",
"lastSeen": "2019-05-18T19:12:38Z",
"locale": "en-US",
"userId": "ABwppHHd40lIZ1o0bRERAKlHNtNcS2qFtz7NbRQnb31AQDFuV41VPFQivXwwpQGtv_5SlsZNp0N3kxalIIXXXXXX"
},
"conversation": {
"conversationId": "ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX",
"type": "NEW"
},
"availableSurfaces": [
{
"capabilities": [
{
"name": "actions.capability.AUDIO_OUTPUT"
},
{
"name": "actions.capability.WEB_BROWSER"
},
{
"name": "actions.capability.SCREEN_OUTPUT"
}
]
}
]
}
},
"session": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX"
}
Most notable from that is the session attribute set at the bottom from that. The code uses this as the parent and session values when building the
The request objects for the other two Intents are similar, and all have the same value for session. None of the response objects are notable in any way.
When the Welcome Intent is triggered, the call to listSessionEntities(), unsurprisingly, shows there aren't any yet:
info: listSessionEntities parent projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX
info: listSessionEntities [
[],
null,
null
]
The parent appears to have the correct value from the session however.
When the handler for the Welcome Intent goes and creates the Session Entity, things appear to work ok:
info: setSessionEntity request {
"parent": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX",
"sessionEntityType": {
"displayName": "code",
"entities": [
{
"value": "alpha",
"synonyms": [
"alpha"
]
},
{
"value": "bravo",
"synonyms": [
"bravo"
]
}
],
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/entityTypes/code",
"entityOverrideMode": "ENTITY_OVERRIDE_MODE_OVERRIDE"
}
}
info: setSessionEntity created [
{
"entities": [
{
"synonyms": [
"alpha"
],
"value": "alpha"
},
{
"synonyms": [
"bravo"
],
"value": "bravo"
}
],
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/entityTypes/code",
"entityOverrideMode": "ENTITY_OVERRIDE_MODE_OVERRIDE"
},
null,
null
]
The parent appears to be the same as the session, and the name appears to follow the correct format, including the additional part that has the "/entityTypes/" followed by the display name.
When I try calling it with the code "alpha", which should trigger the "entry" Intent, it instead triggers the Fallback Intent. The call to listSessionEntities() seems to show the "code" Entity with the Entity Types we expect, even tho there was no match for "alpha".
info: listSessionEntities parent projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX
info: listSessionEntities [
[
{
"entities": [
{
"synonyms": [
"alpha"
],
"value": "alpha"
},
{
"synonyms": [
"bravo"
],
"value": "bravo"
}
],
"name": "projects/session-test-XXXXX/agent/sessions/ABwppHFGTpcFtHOOo6ehQfKys9AnH14V5-RhzrNKsea6y6L5zgv4eN-j3IesfuqSsKMc7qRt1CAOhkUYA9XXXXXX/entityTypes/code",
"entityOverrideMode": "ENTITY_OVERRIDE_MODE_OVERRIDE"
}
],
null,
null
]
Again, everything looks correct. When I try it again with "zulu", it shows the same thing for the call to listSessionEntites(), but this time it matches the "entry" Intent, since "zulu" is one of the Entity Types for "code" that is defined as a Developer Entity.
This is where I am stuck. Everything looks correct. It looks like the Session Entity should be set correctly for this session. It looks like it should be using those values. But it never seems to do so.
What is going on? All help would be greatly appreciated. (Did you even read till the end of the question? If so - thank you! I know it's long, but wanted to be as complete as possible.)
This appears to be a bug - I've gotten feedback from other developers that they're seeing the same problem, sometimes on previously working code.
A bug has been opened at https://issuetracker.google.com/issues/133166381 to track the issue. Star it to indicate you have similar problems and to track progress.
Instantiating a SignIn using conv.ask() always fails. My action says no-response when sign-in flow is called and it leaves my app.
In my Firebase Functions log, the json below was spitted, which probably means that it is correctly called, but there is something wrong. I tried updating my Actions on Google console, cleared sign in setting in the Directory page, tested in some different accounts, cleared data to reset in my action app and as such. Needless to say, I set clientId inside the dialogflow class paramerter.
I totally have no idea for this. I have done many projects for Google Assistant apps which required SignIn Account Linking flow. But I encounter this strange behavior only for this project. What should I do? SDK verions is "actions-on-google": "^2.6.0".
Response {
"status": 200,
"headers": {
"content-type": "application/json;charset=utf-8"
},
"body": {
"payload": {
"google": {
"expectUserResponse": true,
"systemIntent": {
"intent": "actions.intent.SIGN_IN",
"data": {
"#type": "type.googleapis.com/google.actions.v2.SignInValueSpec",
"optContext": "For account creation"
}
},
"richResponse": {
"items": [
{
"simpleResponse": {
"textToSpeech": "Please do sign in "
}
}
]
}
}
}
}
}
Inside conv.userγ
{
"raw":{
"locale":"ja-JP",
"userId":"ABwppHGDYgpYn2waeaDyKoMZIqhg4ZxO56vXeAhs-43ZUC6VF3SdCvUGY3L1Vd08eTY_hRLfjrvp4"
},
"storage":{
},
"_id":"ABwppHGDYgpYn2waeaDyKoMZIqhg4ZxO56vXeAhs-43ZUC6VF3SdCvUGY3L1Vd08eTY_hRSfjrvp4",
"locale":"ja-JP",
"permissions":{
},
"last":{
},
"name":{
},
"entitlements":{
},
"access":{
},
"profile":{
}
}
// THIS SEEMS TO BE CALLED according to Firebase Functions Log
app.intent('StartSignIn', (conv) => {
conv.ask(new SignIn('For account creation'));
})
app.intent(INTENT.SignInIntent, (conv, params, signin) => {
console.log('INTENT.SignInIntent is called');
if (signin.status !== 'OK') {
conv.close(`failed`);
return;
}
conv.close('please wait');
});
UPDATE
I figured out the solution by making another folder and doing firebase init, and connected it to a new Firebase project. The main reason is unknown but it worked well by creating a new project.
I'm building a multi-language-educational app on DialogFlow.
and i hit a big problem : Switch recognition language.
Any suggestions how could i achieve that or any other approaches?
I have added to DialogFlow web-interface secondary language.
I'm using action-on-google sdk with node.js webhook.
UPDATE :
let responseToUser = {
//fulfillmentMessages: richResponsesV2, // Optional, uncomment to enable
//outputContexts: [{ 'name': `${session}/contexts/weather`, 'lifespanCount': 2, 'parameters': {'city': 'Rome'} }], // Optional, uncomment to enable
fulfillmentText: 'This is from Dialogflow\'s Cloud Functions for Firebase editor! :-)', // displayed response
payload: {
audioConfig : {
"audioEncoding": 'AUDIO_ENCODING_FLAC',
"sampleRateHertz": 16000,
"languageCode": 'de',
"phraseHints": ["gutten tag"]
}
},
function sendResponse (responseToUser) {
// if the response is a string send it as a response to the user
if (typeof responseToUser === 'string') {
let responseJson = {fulfillmentText: responseToUser}; // displayed response
response.json(responseJson); // Send response to Dialogflow
} else {
// If the response to the user includes rich responses or contexts send them to Dialogflow
let responseJson = {};
// Define the text response
responseJson.fulfillmentText = responseToUser.fulfillmentText;
responseJson.payload = responseToUser.payload;
// Send the response to Dialogflow
console.log('Response to Dialogflow: ' + JSON.stringify(responseJson));
response.json(responseJson);
The request json :
{
"responseId": "c4ea35d3-1455-4142-b7d2-22417c5880ae",
"queryResult": {
"queryText": "hi",
"action": "default",
"parameters": {},
"allRequiredParamsPresent": true,
"fulfillmentText": "This is from Dialogflow's Cloud Functions for Firebase editor! :-)",
"fulfillmentMessages": [
{
"text": {
"text": [
"This is from Dialogflow's Cloud Functions for Firebase editor! :-)"
]
}
}
],
"webhookPayload": {
"sampleRateHertz": 16000,
"languageCode": "de",
"phraseHints": [
"gutten tag"
],
"audioEncoding": "AUDIO_ENCODING_FLAC"
},
"outputContexts": [
{
"name": "projects/homehf-master/agent/sessions/824b34b9-45f8-4c65-9377-a31242d3414b/contexts/test_context",
"lifespanCount": 5
}
],
"intent": {
"name": "projects/homehf-master/agent/intents/0259f134-6d4d-4aed-920d-9240adbe38fe",
"displayName": "starter_intent"
},
"intentDetectionConfidence": 0.75,
"diagnosticInfo": {
"webhook_latency_ms": 44
},
"languageCode": "en"
},
"webhookStatus": {
"message": "Webhook execution successful"
}
}
Dialogflow requires a language with the audio request, if you'd like to change that language you'll need to issue a new request. You can implement some logic in your webhook and frontend to try the request again with a different language.
To do this add some data in the payload field of your webhook response response indicating that you'd like to issues a new request with the a different language specified. In your client, check the data in the response payload and if the data indicates retrying with another language, send another request with a different language specified in the input audio config with the same audio.