I have a Cloud Function written in the Node.js v8 that uses the #google-cloud/bigquery v1.3.0 library.
I like it, I'm able to perform BigQuery changes such as creating a view using the very simple code below without worry about promises and it's synchronous.
const bigquery = new BigQuery({projectId: 'my-project'});
const options = {
view: {
query: 'SELECT * FROM `my-project.my-datatset.my-table`',
useLegacySql: false
}
};
results = await bigquery
.dataset('my-datatset')
.createTable('my-view', options);
But I've been unable to work out how this code can be modified to perform a patch operations. I would expect a very similar syntax to be available but I can't find it. E.g. none of the examples below work:
//bigquery.dataset(datasetId).patchTable(viewId,options);
//bigquery.dataset(datasetId).table(viewId).patch(options);
//bigquery.dataset(datasetId).tables(viewId).patch(options);
I'm able to do the patch operation I want using the rest API through Googles reference documents. But I just can't find a code solution that's consistent with the approach above.
Any ideas?
This solution is longer and asynchronous, but it seems to work. In case anyone runs into the same problem
var {google} = require('googleapis');
var bigQuery = google.bigquery("v2")
google.auth.getApplicationDefault(function(err, authClient) {
if (err) {
//Handle error
}
if (authClient.createScopedRequired && authClient.createScopedRequired()) {
var scopes = [
//Either scope is sufficient according to the spec.
//https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/patch
'https://www.googleapis.com/auth/cloud-platform',
'https://www.googleapis.com/auth/bigquery'
];
authClient = authClient.createScoped(scopes);
}
var request = {
projectId: 'my-project',
datasetId:'my-datatset',
tableId:'my-view',
resource: {
view: {
query: 'SELECT * FROM `my-project.my-datatset.my-table`',
useLegacySql: false
}
},
// Auth client
auth: authClient
};
tables = bigQuery.tables;
tables.patch(request, function(err, response) {
if (err) {
//Handle error
} else {
//Print response
}
});
});
Related
I have a fairly straightforward script that reads summary data from an api and the loops through the records to save the detail to a database.
The code runs without problems when I launch it from VS Code but when I move it into a Lambda function it only runs halfway through.
There are two api calls using axios. The first gets the summary and the second pulls the detail.
The first call works in Lambda. The second, which uses the same method, does not. I can tell through logging statements that the correct data is getting to the second method. The only real differences are that the second is in a loop and it also uses Bottleneck to prevent overloading a touchy api.
I have put logging statements all over the place but once the routine enters the second api call I get no response at all. The logging statement directly inside the routine shows that it is getting there but I don't get anything back from axios. No success or error.
Here is the code.
var Bottleneck = require("bottleneck");
const axios = require('axios');
const Sequelize = require('sequelize');
let apiKey = process.env.APIKEY;
var timeDelay = 1000;
const instance = axios.create({
baseURL: 'https://anapi.com/api/v1/',
headers: {
'Content-Type': "application/json",
'X-Auth-Key': apiKey,
}
});
const limiter = new Bottleneck({
maxConcurrent: 1,
minTime: timeDelay
});
const sequelize = new Sequelize(
"postgres://postgres:reallystrongpassword#awsrdsdb.cluster-vms39sknjssk1.us-west-2.rds.amazonaws.com/targetdatabase"
);
const notes = sequelize.define(
"notes",
{
appointmentid: {
type: Sequelize.STRING,
}, ...
questions: {
type: Sequelize.JSONB,
},
},
{
tableName: "notes",
timestamps: false
}
);
async function notesInject(detailData) {
log.info("inside notesInject");
const injector = await notes.create({
appointmentid: detailData.AppointmentId,
...
questions: detailData.Questions,
}).then(function(){
log.info("created note ", detailData.Id)
}).catch(function(error){
log.info(error)
})
}
function getDetail(detailId) {
log.info(detailId)
try {
instance.get('notes/' + detailId)
.then ((resp) => {
try {
var detailData = (resp.data)
} catch {
log.info("detailData success", resp.status)
}
try {
notesInject(detailData)
} catch (error) {
log.info("notesInject catch", resp.status);
}
})
} catch (error) {
log.info("error in the detail instance")
}
}
function procDetail(apiData) {
for (let i = 0; i < apiData.length; i++) {
const element = apiData[i];
let detailId = element.Id;
getDetail(detailId)
}
}
function getTodayData() {
const pullDate = new Date();
const dateY = pullDate.getFullYear();
const dateM = pullDate.getMonth()+1;
const dateD = pullDate.getDate()-1;
const apiDate = (dateY+'-'+dateM+'-'+dateD)
try {
instance.get('notes/summary?startDate=' + apiDate)
.then ((resp) => {
try {
var apiData = (resp.data)
} catch {
log.info("set apiData", resp.status)
}
try {
procDetail(apiData)
} catch (error) {
log.info("saveDetail", resp.status);
}
})
} catch (error) {
log.info("in the summary instance")
}
}
exports.handler = async (event) => {
getTodayData();
};
I was thinking that the problem was with Bottleneck because that is the most significant difference between the first and second axios calls. When I isolated the database write code after the api pull, it had the same behavior. No error or response.
I'm stumped. I've logged everything I can think of. Lambda doesn't display console.log messages for this so I've been using Lambda-Log.
I'm sure it's something dumb but it works just fine when I run it from Code.
If anyone has any idea what I'm doing wrong, please let me know.
Sorry if I posted too much code but I really don't know where the problem is.
Many thanks
JS newbie here. I am using Jasmine to test a NodeJS application which uses MongoDB and Mongoose, and I would like to replace my static test fixtures with dynamic factories. https://github.com/ratson/factory-bot looks good to me.
However, all of the examples are from a single file and don't demonstrate exporting/importing between files, so I don't understand what to modules.exports = in order to use a factory in my specs.
I'm also using ES5 if that matters.
My question is: how do I export this definition?
spec/factories/user.js
const factory = require('factory-bot').factory;
factory.setAdapter(new FactoryBot.MongooseAdapter());
const User = require('../models/user');
factory.define('user', User, {
username: 'Bob',
expired: false
});
factory.extend('user', 'expiredUser', {
expired: true
});
And then how do I use my export so that I can make sampleUsers?
spec/controllers/user.js
const reqs = require("../support/require")
describe("GET /users", () => {
describe("index", () => {
var data = {};
var sampleUsers = factory.createMany('user', 5);
beforeEach((done) => {
reqs.Request.get(/users", (error, response, body) => {
data.status = response.statusCode;
data.body = JSON.parse(body);
done();
});
});
it("returns a 200 response status", () => {
expect(data.status).toBe(200);
});
it("responds with the users collection", async () => {
expect(data.body.users).toBe(sampleUsers);
});
});
});
Thanks in advance for any advice.
You just need to require your factory definitions before using them.
Here's an example of what you could do:
spec/factories/user.js
const { factory } = require('factory-bot');
const User = require('../models/user');
factory.setAdapter(new FactoryBot.MongooseAdapter());
factory.define('user', User, {
username: 'Bob',
expired: false
});
factory.extend('user', 'expiredUser', {
expired: true
});
spec/factories/index.js:
const { factory } = require("factory-bot");
// Require factories to use with the exported object
require("./user.js");
module.exports = factory;
spec/controllers/user.js:
const factory = require("../../factories");
...
const sampleUsers = factory.createMany('user', 5);
The key difference between the example above and your sample code is the index.js file which requires factory-bot and all the factory definitions. By requiring the definitions, you will be able to use them.
If you require('factory-bot') directly instead of require('spec/factories'), you will need to require the factory definitions you want to use.
I'm trying to create 2 calendar events across 2 calendars using the Google Node.js API client library in a single function. I'm using 2 separate auth objects that I'm retrieving like so:
var auth1 = await google.auth.getClient({ credentials: config.account1Creds, scopes: ["https://www.googleapis.com/auth/calendar.events"] });
var auth2 = await google.auth.getClient({ credentials: config.account2Creds, scopes: ["https://www.googleapis.com/auth/calendar.events"] });
I can create the event on the first calendar fine, but when I create the event on the 2nd calendar, I get a 404 Not Found back from the Google Calendar API server.
If I comment out the first var auth1 = await google.auth... line and only create the event on the 2nd calendar, everything is fine and the event is created successfully.
It kinda feels as if the first time getClient is called, it sets a global auth object that gets used for all remaining API requests and it can't be replaced, but thats just a hunch I have and I don't actually know.
Anybody have any ideas why this is happening?
EDIT:
GetGoogleCalendarService: async function(credentials)
{
var auth = await google.auth.getClient({ credentials: credentials, scopes: ["https://www.googleapis.com/auth/calendar.events"] });
return google.calendar({ version: "v3", auth: auth });
},
InsertCalendarEvent: function(calendar, entry, callback)
{
calendar.events.insert(entry, callback);
},
SendInvitesToEvent: async function (request, response)
{
//build the calendar event
var entry = {
...
}
//insert into operations calendar
var opsCal = await Events.GetGoogleCalendarService(config.GetConfig().OpsCalendarCredentials);
Events.InsertCalendarEvent(mainCal, entry);
//insert into public calendar
var publicCal = await Events.GetGoogleCalendarService(config.GetConfig().PublicCalendarCredentials);
Events.InsertCalendarEvent(publicCal, entry, async function(err, event)
{
//err: 404 Not Found
...
}
}
You want to insert events to 2 calendars using 2 clients.
Client "A" inserts an event to Calendar "A".
Client "B" inserts an event to Calendar "B".
You are using googleapis of Node.js.
If my understanding is correct, how about this modification? In this modification, I separated retrieving auth by each credential.
Modified script:
const { google } = require("googleapis");
function insertEvent(calendar, calendarId) {
// insert event
}
async function getService(c) {
var auth = await google.auth.getClient({
credentials: c,
scopes: ["https://www.googleapis.com/auth/calendar.events"]
});
return google.calendar({ version: "v3", auth: auth });
}
function main() {
getService(config.account1Creds).then(calendar => {
insertEvent(calendar, "### calendarId ###");
});
getService(config.account2Creds).then(calendar => {
insertEvent(calendar, "### calendarId ###");
});
}
main();
Note:
This is a sample script. So please modify this for your situation.
In my environment, I could confirm that this script works. But if this didn't work and this modification was not the result you want, I apologize.
Edit:
From your current script, I modified as follows.
Is opsCal mainCal? In my modification, opsCal is used as mainCal.
Modified script:
From:
//insert into operations calendar
var opsCal = await Events.GetGoogleCalendarService(config.GetConfig().OpsCalendarCredentials);
Events.InsertCalendarEvent(mainCal, entry);
//insert into public calendar
var publicCal = await Events.GetGoogleCalendarService(config.GetConfig().PublicCalendarCredentials);
Events.InsertCalendarEvent(publicCal, entry, async function(err, event)
{
//err: 404 Not Found
...
}
To:
Events.GetGoogleCalendarService(config.GetConfig().OpsCalendarCredentials).then(
opsCal => {
Events.InsertCalendarEvent(opsCal, entry);
}
);
Events.GetGoogleCalendarService(config.GetConfig().PublicCalendarCredentials).then(
publicCal => {
Events.InsertCalendarEvent(publicCal, entry);
}
);
I have a working code to fire query on hive using node module "jdbc", as an alternative i was trying "jshs2" , i am able to connect to hive and fire query but still stuck to retrieve the resultset, can anyone who have used the "jshs2", module can put up an example.
Thanks for any help.
I just started a project where I have to connect to hive from node too. I was able to run a query on database and iterate through the resultset using the following demo code:
const {
Configuration,
HiveConnection,
IDLContainer,
} = require('jshs2');
const options = {
auth: 'NOSASL',
host: 'myServer',
port: myPort,
};
const hiveConfig = new Configuration(options);
const idl = new IDLContainer();
async function main() {
await idl.initialize(hiveConfig);
const connection = await new HiveConnection(hiveConfig, idl);
const cursor = await connection.connect();
const res = await cursor.execute('SELECT * FROM orders LIMIT 10');
if (res.hasResultSet) {
const fetchResult = await cursor.fetchBlock();
fetchResult.rows.forEach((row) => {
console.log(row);
});
}
cursor.close();
connection.close();
}
main().then(() => {
console.log('Finished.');
});
I'm using node v8.x, so I can use ES6 features like destructuring and async/await. If your node version is older you must work with promise chains.
I have an API that searches for the user-provided term, returns an array of results, then fires off async requests for each of the results and gets results for each of these second batch of requests. I'd like the API to report progress as it happens rather than just the final result. So, if I do the following request, I should get updates like so
$ curl 'http://server/?q=foobar'
searching for ${q}…
found 76… now getting images…
found 30 images… done
{
result
}
Most of relevant code is shown below. Fwiw, I am using hapijs for my application.
let imagesOfRecords = {};
const getImages = async function (q) {
console.log(`searching for ${q}…`);
const uri = `http://remoteserver/?q=${q}`;
const {res, payload} = await Wreck.get(uri);
const result = JSON.parse(payload.toString()).hits;
const numOfFoundRecords = result.total;
if (result.total) {
console.log(`found ${result.total}… now getting images…`);
const foundRecords = result.hits.map(getBuckets);
Promise.all(foundRecords).then(function() {
console.log(`found ${Object.keys(imagesOfRecords).length} images… done`);
reply(imagesOfRecords).headers = res.headers;
}).catch(error => {
console.log(error)
});
}
else {
console.log('nothing found');
reply(0).headers = res.headers;
}
};
const getBuckets = async function(record) {
const { res, payload } = await Wreck.get(record.links.self);
const bucket = JSON.parse(payload.toString()).links.bucket;
await getImageFiles(bucket, record.links.self);
};
const getImageFiles = async function(uri, record) {
const { res, payload } = await Wreck.get(uri);
const contents = JSON.parse(payload.toString()).contents;
imagesOfRecords[record] = contents.map(function(el) {
return el.links.self;
});
};
Once I can implement this, my next task would be to implement this progressive update in a web app that uses the above API.
To show result with each step of your requests for backend you can use EventEmitter, which will emit event on each progress step. You can read about events here.
Simple implementation:
const events = require('events');
const eventEmitter = new events.EventEmitter();
//your request code
Promise.all(foundRecords).then(function() {
console.log(`found ${Object.keys(imagesOfRecords).length} images… done`);
eventEmitter.emit('progress');
reply(imagesOfRecords).headers = res.headers;
})
const eventReaction = (e) => {
// do something with event, console log for example.
}
eventEmitter.on('progress', eventReaction);
More examples you can find here and here.
To show events to client you can use library socket.io. I think you can find pretty straightforward explanations how socket.io works in documentation.
If you want to send events between servers or processes and want to go little further, you can read more about 0MQ (zero mq) and it's node implementation