I'm trying to work with CosmosDB in Nodejs. I have created some Documents in a collection and have added a couple attachments to one of the documents like so:
let dbds = new azure(dbEndpoint, {"masterKey":dbKey})
fs.open("c:/temp/capture.png", "r", (err, fd) => {
if(err) console.log(err);
else{
dbds.createAttachmentAndUploadMedia(documentLink, fs, {contentType:"image/png", partitionKey: partitionKey}, (err, result) =>{
fs.close(fd);
if(err) console.log(err);
else console.log(result);
}
}
}
Now when I read that attachments for that document, I get 2 attachments:
dbds.readAttachments(documentLink, {"partitionKey":partitionKey}).toArray((err, result) => {
if(err) console.log(err);
else {
console.log(result.length); //2
result.forEach(i => console.log(i.id);) //id of each attachment
}
}
Now I need to be able to read the document back and store it locally. There doesn't seem to be any documentation that I can find for doing this in Nodejs. I've tried the following:
let attachment = result[0]; //from result of toArray() above -- first attachment from list
let mediaURI = `${attachment._self}${attachment.media}`
dbds.readMedia(mediaURI, (err, result) => {
if(err) console.log(JSON.parse(err.body).message); //Request url is invalid
else {
//try to write result to file
}
}
How do I create a valid URI to the media in order to download it?
Edit
Based on comments below, I updated my code as follows:
let attachment = result[0]
let mediaURI = attachment.media //here is the change
dbds.readMedia(mediaURI, (err, result) => {
if(err) console.log(JSON.parse(err.body).message);
else {
//try to store the data
}
})
I no longer get an error but instead get a JSON object:
"{"constants":{"O_RDONLY":0,"O_WRONLY":1,"O_RDWR":2,"S_IFMT":61440,"S_IFREG":32768,"S_IFDIR":16384,"S_IFCHR":8192,"S_IFLNK":40960,"O_CREAT":256,"O_EXCL":1024,"O_TRUNC":512,"O_APPEND":8,"F_OK":0,"R_OK":4,"W_OK":2,"X_OK":1},"F_OK":0,"R_OK":4,"W_OK":2,"X_OK":1}"
I got help from some folks at Microsoft on this.
The primary issue was that I wasn't writing the file correctly.
Instead of this
let dbds = new azure(dbEndpoint, {"masterKey":dbKey})
fs.open("c:/temp/capture.png", "r", (err, fd) => {
if(err) console.log(err);
else{
dbds.createAttachmentAndUploadMedia(documentLink, fs, {contentType:"image/png", partitionKey: partitionKey}, (err, result) =>{
fs.close(fd);
if(err) console.log(err);
else console.log(result);
}
}
}
With fs.open fs is not a filestream -- which is what createAttachmentAndUploadMedia is looking for. I had mistakenly assumed that fs was a fs.open created a filestream.
What I wound up doing was the following:
fs.readFile("c:/temp/myimage.jpg",(err, data) => {
if(err) reject(err);
else dbds.svc.createAttachmentAndUploadMedia(docPath,data,{contentType: "image/jpeg"
, partitionKey: "Key"}, (err, result) =>{..}
})
In your example, the mediaURI should just be ${attachment.media} not concatenated with the ${attachment._self}.
Here is an snippet:
return new Promise<any>((resolve, reject) => {
this.documentDbClient.readAttachment(docLink, options, (err, result) => {
if (err)
return reject(err);
resolve(result.media);
});
}).then((medialink) => {
return new Promise<any>((resolve, reject) => {
this.documentDbClient.readMedia(medialink, (err, result) => {
if (err)
return reject(err);
// buffered data
const data = [];
result.on('data', (chunk) => data.push(chunk));
result.on('end', () => {
resolve(data.join(''));
});
// if not buffered
// resolve(result.toString());
});
});
});
Related
In my nodejs project, there's a function where I have two queries to get data from. First query is dependent on a flag, if flag is true, then I've to run that query and pass its data to second query. Otherwise I've to set some hard-coded data for second query. But it does not wait for first query response and run second query. I've added callbacks to it but nothing worked for me. Here is my code snippet;
getData(req, callback) {
if(!req.flag){
lstData.push('d');// hardcoded data
return Reader.open(filename).then(reader => {
let code = reader.data;
if(!!code ){
return dao.getFirstData(code , (err, results) => {
if (err) return callback(err);
if(results && results.length > 0){
return lstData = results;
}
});
}
});
}
else{
lstData.push('d');// hardcoded data
}
let asyncFuncs = [
(callback) => {
dao.getSecondData(lstData, (err, results) => {
if (err) return callback(err);
return callback(null, { amounts: results });
});
}
];
asyncFuncs.push(callback => {
dao.thirdFunction(id, callback);
});
async.parallel(asyncFuncs, (err, results) => {
if (err) return callback(err);
let data1= results[0].amount;
let data2= results[1];
// some calculations with data1 & data2
return callback(err, finalResult);
});
}
No matter flag is true or false, getSecondData always returns data against d (hard-coded value). I am a newbie to nodejs, so please let me know what am I doing wrong.
SO I updated my code to add promise in it...here is updated code;
getData(req, callback) {
if(!req.flag){
lstData.push('d');// hardcoded data
Reader.open(filename).then(reader => {
let code = reader.data;
if(!!code ){
var prom = new Promise((resolve, reject) => {
dao.getFirstData(code , (err, results) => {
if (err) return callback(err);
if(results && results.length > 0){
let lstData = results;
return resolve(lstData);
}
});
});
prom.then((result) => {
return result;
});
}
});
}
else{
lstData.push('d');// hardcoded data
}
let asyncFuncs = [
(callback) => {
dao.getSecondData(lstData, (err, results) => {
if (err) return callback(err);
return callback(null, { amounts: results });
});
}
];
asyncFuncs.push(callback => {
dao.thirdFunction(id, callback);
});
async.parallel(asyncFuncs, (err, results) => {
if (err) return callback(err);
let data1= results[0].amount;
let data2= results[1];
// some calculations with data1 & data2
return callback(err, finalResult);
});
}
But still same response. It is not waiting for promise result.
There plenty of ways to achieve this but I would prefer http://async.io/ to perform multi function operations.
By using promise you can do,
function f1(argument){
return new Promise((resolve, reject) => {
try {
data = // do your thing
resolve(data)
}
catch(err){
reject(err)
}
})
}
and then use await to achieve this
async function caller(){
await f1()
}
Few other links to help you [here][2]
[2]: https://stackoverflow.com/questions/22442321/callback-function-example/48333938#:~:text=A%20callback%20function%2C%20is%20a,or%20executed)%20inside%20the%20otherFunction.&text=Without%20thinking%20too%20much%2C%20see%20the%20following%20example.&text=Generally%2C%20JavaScript%20allows%20function%20as%20a%20parameter.
You can use async/await for asynchronous calls or you can uses promises. This will wait at the asynchronous calls and once you get the response your code will execute the dependent subsequent calls.
getData(req, callback) {
if(!req.flag){
lstData.push('d');// hardcoded data --> this might be an issue as you are already pushing data.
// these are async calls where you can ask your code to wait
return Reader.open(filename).then(reader => {
let code = reader.data;
if(!!code ){
var promise = dao.getFirstData(code , (err, results) => {
if (err) reject(err);
if(results && results.length > 0){
lstData = results;
resolve(lstData);
}
});
return promise --> you can subscribe this using then
}
});
}
else{
lstData.push('d');// hardcoded data
}
let asyncFuncs = [
(callback) => {
dao.getSecondData(lstData, (err, results) => {
if (err) return callback(err);
return callback(null, { amounts: results });
});
}
];
}
To get hands on this you can check the ways of using async/await
https://javascript.info/async-await
Swagger Inspector
This is the source code
API.js
router.post('/curriculum2_section/', (req,res)=> {
curriculum2_section.CREATE(req.body.curr2_section,req.body.curr2_id,req.body.curr2_section_student_amount)
})
curriculum2_section.js
module.exports.CREATE = function CREATE(curr2_section,curr2_id,curr2_section_student_amount) {
var sqlString = `INSERT INTO curriculum2_section(curr2_id,curr2_section,curr2_section_student_amount)VALUES('${curr2_id}',${curr2_section},${curr2_section_student_amount})`
pool.getConnection((err, conn) => {
if(err) throw err
pool.query(sqlString, function (err, rows) {
console.log('Query String:\n '+this.sql);
if(err) throw err
console.log('Data Insert: '+curr2_section+','+curr2_id+','+curr2_section_student_amount)
conn.release();
})
})
}
I've tried using the callback but it doesn't work. I think it because I don't understand how to use it
Please help me
You need to pass the data you want to output to a method on res in order to get output from Express.
Given what you have here, your best bet might be to add a callback as the last parameter of curriculum2_section.CREATE and use that callback to pass data to res.
https://expressjs.com/en/4x/api.html#res
router.post('/curriculum2_section/', (req,res)=> {
curriculum2_section.CREATE(req.body.curr2_section,req.body.curr2_id,req.body.curr2_section_student_amount, (err, data) => {
res.send(data);
});
});
module.exports.CREATE = function CREATE(curr2_section,curr2_id,curr2_section_student_amount, cb) {
var sqlString = `INSERT INTO curriculum2_section(curr2_id,curr2_section,curr2_section_student_amount)VALUES('${curr2_id}',${curr2_section},${curr2_section_student_amount})`
pool.getConnection((err, conn) => {
if(err) throw err
pool.query(sqlString, function (err, rows) {
console.log('Query String:\n '+this.sql);
if(err) throw err
console.log('Data Insert: '+curr2_section+','+curr2_id+','+curr2_section_student_amount)
conn.release();
cb(null, 'Data Insert: '+curr2_section+','+curr2_id+','+curr2_section_student_amount);
})
})
}
I have a very large multi section script with a LOT of loops and some recursion in it. When I run it on a Very Large dataset, the script will simply stop running. It stops with a 0 exit code. It VERY clearly does not actually finish running...it just...stops.
asyncLib.waterfall([
getPronghornToken,
saveSchedulers,
saveServices,
populateServRefs,
saveServiceGroups,
saveNetworks,
populateNetRefs, //never actually gets out of this function. Just exits with code 0
saveNetworkGroups,
saveRuleGroups,
fetchRuleGroupIds,
populateRules,
saveRules,
getPolicyId,
linkRuleGroup
], function (err, result) {
if (err){
console.error("Something bad happened. Please try again");
process.exit(1);
}
console.log("done");
});
What I'm looking for: Why would a script just stop mid loop and exit with a 0 code?
Note: Alternate code.
getPronghornToken((err, token) => {
if(err) {
console.log("Error occured getPronghornToken");
throw err;
}
saveSchedulers(token, (err, token) => {
if(err) {
console.log("Error occured saveSchedulers");
throw err;
}
saveServices(token, (err, token) => {
if(err) {
console.log("Error occured saveServices");
throw err;
}
populateServRefs(token, (err, token) => {
if(err) {
console.log("Error occured populateServRefs");
throw err;
}
saveServiceGroups(token, (err, token) => {
if(err) {
console.log("Error occured saveServiceGroups");
throw err;
}
saveNetworks(token, (err, token) => {
if(err) {
console.log("Error occured saveNetworks");
throw err;
}
populateNetRefs(token, (err, token) => {
if(err) {
console.log("Error occured populateNetRefs");
throw err;
}
saveNetworkGroups(token, (err, token) => {
if(err) {
console.log("Error occured saveNetworkGroups");
throw err;
}
saveRuleGroups(token, (err, token) => {
if(err) {
console.log("Error occured saveRuleGroups");
throw err;
}
fetchRuleGroupIds(token, (err, token) => {
if(err) {
console.log("Error occured fetchRuleGroupIds");
throw err;
}
populateRules(token, (err, token) => {
if(err) {
console.log("Error occured populateRules");
throw err;
}
saveRules(token, (err, token) => {
if(err) {
console.log("Error occured saveRules");
throw err;
}
getPolicyId(token, (err, token) => {
if(err) {
console.log("Error occured getPolicyId");
throw err;
}
linkRuleGroup(token, (err, token) => {
if(err) {
console.log("Error occured linkRuleGroup");
throw err;
}
console.log("Successfully installed all files");
});
});
});
});
});
});
});
});
});
});
});
});
});
});
No errors thrown. Does NOT print out the innermost message. Callback pattern verified.
Last Function running looks like this:
async function populateNetRefs(token, callback) {
//let newNetRefs = [];
for(let index = 0; index < networkGroups.length; index++) {
if (index >= networkGroups.length) {
console.log("Net Refs Finished")
return callback(null, token);
}
let networkGroup = networkGroups[index];
try {
console.log(`fetching network number: ${index+1} / ${networkGroups.length}`);
let newNetRefs = await fetchNetId(token, networkGroup._netRefs);
networkGroup._netRefs = newNetRefs;
} catch (err) {
console.log(`An error occurrent fetching the network id for index ${index+1} / ${networkGroups.length}: ${err}`);
}
}
}
The Inner Function:
function fetchNetId(token, _netRefs) {
let fetchFinished = 0;
let newNetRefs = [];
let errCount = 1;
console.log("ZZ Fetchid Start ZZ");
return new Promise((resolve, reject) => {
_netRefs.forEach(function(_netRef) {
let options = {
//Required to be hidden
};
let req = https.request(options, (res) => {
let reply = [];
res.setEncoding('utf8');
res.on('data', (chunk) => {
console.log("YY GET DATA CHUNK YY");
reply.push(chunk);
});
res.on('end', () => {
fetchFinished++;
console.log("Reply is : " + reply.join());
//There is some logic in this spot. Not for you.
console.log("fetchFinished is: " + fetchFinished + ", size is: " + _netRefs.length);
if (fetchFinished === _netRefs.length) {
resolve(newNetRefs);
}
});
});
req.on('error', (e) => {
console.error(`problem with request ${errCount++}: ${e.message}`);
//reject(e);
});
let body = JSON.stringify({
"options" : {
"start": 0,
"limit": 5,
"sort": {
},
"filter": {
"name":{"$eq":_netRef}
}
}
});
console.log("XX Sending Request XX");
req.write(body);
req.end();
});
});
}
BEHAVIOR UPDATE - More Console Logs
Here's the end of the console log:
fetching network number: 49 / 711
ZZ Fetchid Start ZZ
XX Sending Request XX
XX Sending Request XX
YY GET DATA CHUNK YY
Reply is : {hidden from you}
TroubleShootingDias: some guid
fetchFinished is: 1, size is: 2
YY GET DATA CHUNK YY
Reply is : {hidden from you}
TroubleShootingDias: some guid
fetchFinished is: 2, size is: 2
fetch success
fetching network number: 50 / 711
ZZ Fetchid Start ZZ
[vagrant#vag-host1 space-parser]$
I highly recommend you look at async.waterfall to help structure code like this as it can be a mare to debug and read. A lot to grasp in your code above, but it could be helpful to wrap the following in a try catch. While you are handling req errors - those are only request errors and there may be something else including possible malformed url etc that will throw and you don't have the promise returning in this instance.
try {
let req = https.request(options, (res) => {
let reply = [];
res.setEncoding('utf8');
res.on('data', (chunk) => {
console.log("YY GET DATA CHUNK YY");
reply.push(chunk);
});
res.on('end', () => {
fetchFinished++;
console.log("Reply is : " + reply.join());
//There is some logic in this spot. Not for you.
console.log("fetchFinished is: " + fetchFinished + ", size is: " + _netRefs.length);
if (fetchFinished === _netRefs.length) {
resolve(newNetRefs);
}
});
});
req.on('error', (err) => {
console.error(`problem with request ${errCount++}: ${err.message}`);
return reject(err);
});
}
catch(err) {
console.error(`problem with request ${err.message}`);
return reject(err);
}
Actual solution to the problem: If the array is empty, then the promise never resolves. Added an empty check to the very top, before the loop.
function fetchNetId(token, _netRefs) {
let fetchFinished = 0;
let newNetRefs = [];
let errCount = 1;
console.log("ZZ Fetchid Start ZZ");
return new Promise((resolve, reject) => {
if(_netRefs.length === 0) return resolve([]) // <==============
_netRefs.forEach(function(_netRef) {
let options = {
//Required to be hidden
};
let req = https.request(options, (res) => {
let reply = [];
res.setEncoding('utf8');
res.on('data', (chunk) => {
console.log("YY GET DATA CHUNK YY");
reply.push(chunk);
});
//........
I need to read a file line by line and write newlines to same file while reading, if each line satisfy certain set of conditions. What could be the best way.
function (file, callback) {
fs.readFile(file, (err, 'utf8', data) => {
if (err) return callback(err);
var lines = data.split('\n');
fs.open(file, 'w', (err, fd) => {
if (err) return callback(err)
lines.forEach(line => {
if (line === 'meet your condition') {
// do your write using fs.write(fd, )
}
})
callback();
})
})
}
use node fs module with the help of fs you can perform operation asynchronously as well as synchronously. below is as an example of asynchronously
function readWriteData(savPath, srcPath) {
fs.readFile(srcPath, 'utf8', function (err, data) {
if (err) throw err;
//Do your processing, MD5, send a satellite to the moon or can add conditions , etc.
fs.writeFile (savPath, data, function(err) {
if (err) throw err;
console.log('complete');
});
});
}
Synchronously example
function readFileContent(srcPath, callback) {
fs.readFile(srcPath, 'utf8', function (err, data) {
if (err) throw err;
callback(data);
}
);
}
function writeFileContent(savPath, srcPath) {
readFileContent(srcPath, function(data) {
fs.writeFile (savPath, data, function(err) {
if (err) throw err;
console.log('complete');
});
});
}
Lets say, for example, I want to write a nodejs program where I have two or three independent parts like fs.readdir, fs.copy, etc. on different locations, but the result all three actions is to be sent to a json file like this:
var fs = require('fs-extra');
var jsd = {
"act1" : false,
"act2" : false,
"act3" : false
}
fs.readdir(path1, function (err, files) {
if (err) jsd.act1 = err;
for (x in files) console.log(files[x]);
jsd.act1 = true;
});
fs.copy(path2, path3, function (err) {
if (err) jsd.act2 = err;
jsd.act2 = true;
});
fs.remove(path4, function (err) {
if (err) jsd.act3 = err;
jsd.act3 = true;
});
// all three of the above actions are independent, so it makes sense that all of them are executed asynchronously.
// Now we write jsd object to a json file; jsd's contents are dependent on the above actions though
fs.writeJson("./data.json", jsd, function (err, files) {
if (err) return console.error(err);
});
How do I make sure that the correct data is entered into the file data.json, i.e fs.writeJson executes after the actions previous to it are executed first?
I know one way is to nest all of them, i.e,
readdir() {
copy() {
remove() {
writeJson();
}
}
}
But this may result in callback hell, so is there a better way to do this?
you can use Promise or module async,
if you use promise, first you must convert all callback function into Promise like this:
const reddir = function(path) {
return new Promise((resolve, reject) => {
fs.readdir(path, (err, files) => {
if (err) return reject(err);
for (x in files) console.log(files[x]);
resolve(true);
});
})
}
then you can use
Promise.all([reddir(path1), copy(path2, path3), remove(path4)])
.spread((act1, act2, act3) => { //.spread is bluebird feature
return writeJson(./data.json);
})
.catch(e => {
// all error can handled in this
})
if you use async module, you can write like this:
async.parallel({
act1: function(cb){
fs.reddir(path1, (err, files) => {
if (err) return cb(err);
for (x in files) console.log(files[x]);
cb(true);
})
},
act2: ...
},(err, jsd) => { // jsd will be {act1: true, act2: ...}
if (err) return console.error(err); // handle all above error here;
fs.writeJson("./data.json", jsd, function (err, files) {
if (err) return console.error(err);
});
})