Parse Google Trends API data into JSON format - node.js
I was working on a program which fetches the data from Google Trends API using Node.js, I am getting the desired data, but not in the correct format. I want to do JSON parsing so that I can get data in the correct format.
My code:
//This program is fetching data from google trend api for a particular job title
var googleTrends = require('google-trends-api');
var fs = require('fs');
fs.readFile('keywords.txt', 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
data = data.toString().split("\n");
recur(0, data);
});
function recur(index, data){
if (index < data.length){
var keyword = data[index].split(",");
console.log(keyword);
googleTrends.interestByRegion({keyword:keyword,
geo:'US',
resolution:'state'
})
.then(function(results ){
console.log(results+ "\n");
console.log("***********************************************************************************************");
index = index+1;
recur(index, data);
})
.catch(function(err){
console.error('We have an error!', err);
});
}
}
Output :-
[ 'House Cleaning' ]
{"default":{"geoMapData":[{"geoCode":"US-CO","geoName":"Colorado","value":[100],"formattedValue":["100"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-AZ","geoName":"Arizona","value":[96],"formattedValue":["96"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-WA","geoName":"Washington","value":[95],"formattedValue":["95"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-KS","geoName":"Kansas","value":[93],"formattedValue":["93"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-NV","geoName":"Nevada","value":[93],"formattedValue":["93"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-ID","geoName":"Idaho","value":[93],"formattedValue":["93"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-RI","geoName":"Rhode
Island","value":[87],"formattedValue":["87"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-UT","geoName":"Utah","value":[86],"formattedValue":["86"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-OR","geoName":"Oregon","value":[85],"formattedValue":["85"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-OK","geoName":"Oklahoma","value":[83],"formattedValue":["83"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-NC","geoName":"North
Carolina","value":[83],"formattedValue":["83"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-TN","geoName":"Tennessee","value":[80],"formattedValue":["80"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-CA","geoName":"California","value":[79],"formattedValue":["79"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-GA","geoName":"Georgia","value":[78],"formattedValue":["78"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-TX","geoName":"Texas","value":[78],"formattedValue":["78"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-MO","geoName":"Missouri","value":[77],"formattedValue":["77"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-MT","geoName":"Montana","value":[77],"formattedValue":["77"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-NH","geoName":"New
Hampshire","value":[76],"formattedValue":["76"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-SC","geoName":"South
Carolina","value":[76],"formattedValue":["76"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-AL","geoName":"Alabama","value":[75],"formattedValue":["75"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-MA","geoName":"Massachusetts","value":[75],"formattedValue":["75"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-FL","geoName":"Florida","value":[75],"formattedValue":["75"],"maxValueIndex":0,"hasData":[true]},{"geoCode":"US-AR","geoName":"Arkansas","value":
Related
Insert new JSON data to the json file using nodejs [duplicate]
I am new to Node.js and JavaScript. I have a results.json file that I want to keep a running log of results from a script that pulls images from the web. However, my current script only overwrites the existing result. How do I build upon or add to the results.json so each subsequent result is logged in the results.json file? I would like it to be valid json.
Here is general example:
var currentSearchResult = someWebSearchResult
var fs = require('fs');
var json = JSON.stringify(['search result: ' + currentSearchResult + ': ', null, "\t");
fs.writeFile("results.json", json);
And the results.json:
[
"search result: currentSearchResult"
]
If you want the file to be valid JSON, you have to open your file, parse the JSON, append your new result to the array, transform it back into a string and save it again.
var fs = require('fs')
var currentSearchResult = 'example'
fs.readFile('results.json', function (err, data) {
var json = JSON.parse(data)
json.push('search result: ' + currentSearchResult)
fs.writeFile("results.json", JSON.stringify(json))
})
In general, If you want to append to file you should use:
fs.appendFile("results.json", json , function (err) {
if (err) throw err;
console.log('The "data to append" was appended to file!');
});
Append file creates file if does not exist.
But ,if you want to append JSON data first you read the data and after that you could overwrite that data.
fs.readFile('results.json', function (err, data) {
var json = JSON.parse(data);
json.push('search result: ' + currentSearchResult);
fs.writeFile("results.json", JSON.stringify(json), function(err){
if (err) throw err;
console.log('The "data to append" was appended to file!');
});
})
Promise based solution [Javascript (ES6) + Node.js (V10 or above)]
const fsPromises = require('fs').promises;
fsPromises.readFile('myFile.json', 'utf8')
.then(data => {
let json = JSON.parse(data);
json.myArr.push({name: "Krishnan", salary: 5678});
fsPromises.writeFile('myFile.json', JSON.stringify(json))
.then( () => { console.log('Append Success'); })
.catch(err => { console.log("Append Failed: " + err);});
})
.catch(err => { console.log("Read Error: " +err);});
If your project supports Javascript ES8 then you could use asyn/await instead of native promise.
I have created data. Js file to maintain data displaying in the table fomat. Including text box to read data from user and how to display data enteted through text box in table
JSFORCE: File is not supported message is being shown while downloading file from sfdc rest api and write on local disk
I have been trying to download image file using JS force for node js and able to create a file on local after retrieving data and converting it to base64 format but image if showing "file not supported message" whereas being able to download javascript type of file with correct data.
I am querying the attachment field of knowledge article in salesforce.
Following is my query :
SELECT Body__c, Attachment__Name__s, Attachment__ContentType__s, Attachment__Length__s, Attachment__Body__s, Id, KnowledgeArticleId, Title, UrlName FROM Knowledge__kav
I am sending GET request to Attachment__Body__s field of article.
Following is my node js code:
function createFile(attachmentBody,attachmntContentType,attachmntName){
var req = {
url: attachmentBody,
method: 'GET',
headers: {
"Content-Type": attachmntContentType
}
};
var test = conn.request(req, function(err, resp) {
if (err) {
console.log(err)
} else {
var fileBuffer=Buffer.from(resp, 'binary').toString('base64');
console.log('fileBuffer--- '+ fileBuffer);
fs.writeFile('./downloadedAttachments/'+attachmntName,fileBuffer,'base64', function(err){
if (err) throw err
console.log('File saved.')
})
}
});
}
Please help me with this.
I am successfully able to download the file in the correct format. following is my updated code :
function createFile(knbid,attachmntName,callback) {
var file_here = conn.sobject('Knowledge__kav').record(knbid);
file_here.retrieve(function (err, response) {
if (err) {
return console.error(err);
callback(0)
} else {
var obj = fs.createWriteStream('./downloadedAttachments/'+attachmntName, {defaultEncoding: 'binary'})
//console.log('blob--'+JSON.stringify(file_here.blob('Attachment__Body__s')));
var stream = file_here.blob('Attachment__Body__s').pipe(obj);
stream.on('finish', function (err, result) {
if (err)
console.log('not downloaded'+knbid);
else
console.log('downloaded-'+knbid);
})
}
});
}
Want to store json data from file to sqlite database but getting error
Error
{ db.transaction(function(tx){ TypeError: db.transaction is not a
function }
I am setting up a question bank backup in my SQLite database so I have question bank in JSON file form. I have written the following code but couldn't get the error as I just started coding with nodejs.
var fs = require("fs");
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database('json.sqlite3');
var file = process.env.CLOUD_DIR + "json.sqlite3";
var exists = fs.existsSync(file);
var jsonfile = {
key: "myvalue"
};
fs.readFile('demo.json', function (err, data) {
if (err) {
return console.error(err);
}
console.log("File content: " + data.toString());
// now save to db
if(!exists) {
console.log("Creating DB file.");
fs.openSync(file, "w");
}
db.serialize(function() {
if(!exists) {
db.run("CREATE TABLE Stuff (thing TEXT)");
}
var jsonString = escape(JSON.stringify(data.toString()));
db.transaction(function(tx){
tx.executeSql('INSERT OR REPLACE INTO Stuff(md5, json,
expires) VALUES ("'+hash+'", "'+jsonString+'","'+expireStamp+'")');
},function(tx,error){
console.log(JSON.stringify(tx.message));
console.log(JSON.stringify(error));
},function(){
});
});
});
db.close();
JSON should be easily updated, edited, and retrieved from the SQLite Database
The error message db.transaction(function(tx){ TypeError: db.transaction is not a function is clear and unambiguous.
Review the API doc for a list of available methods on a Database object.
I have found this tutorial very helpful during the learning process.
Nodejs - "Write After End" error while Exporting from Mongoose
I am trying to write the results of 3 Mongoose find query into a single file test.txt using event-stream nodejs.
My program is something like this…
var es = require('event-stream');
var fs = require('fs');
var wstream = fs.createWriteStream('TestFile.csv');
mongoose.connection.on('connected', function () {
//First Find Query
Model.find({"FirstId":{$exist:true}).cursor()
.pipe(es.map(function (data, callback) {
var csv = json2csv({data:formated, fields:fields, hasCSVColumnTitle:false});
callback(null, csv)
}))
.pipe(wstream);
//Second find query
Model.find({"SecondId":{$exist:true}).cursor()
.pipe(es.map(function (data, callback) {
if(data.Email)
{
var csv = json2csv({data:formated, fields:fields, hasCSVColumnTitle:false});
}
else{
callback();
}
callback(null, csv)
}))
.pipe(wstream);
//Third find query
Model.find({"ThirdId":{$exist:true}).cursor()
.pipe(es.map(function (data, callback) {
if(data.Email)
{
var csv = json2csv({data:formated, fields:fields, hasCSVColumnTitle:false});
}
else{
callback();
}
callback(null, csv)
}))
.pipe(wstream);
});
In this program I am able to write each find query result separately into the file. But when I combine 3-find query in a program, it throws “write after end” error.
Could anyone help me to resolve this error?
Thank You all for your time!
Passing XML File To xml2js With Node.js
Background
I have taken some sample data from a large XML response I will be working with in my app. I decided to save the xml to a response.xml file in my app so I can parse the data without making a bunch of unnecessary request to the api.
I am going to be using xml2js to convert the xml response to a JS Object.
Problem
I have been able to open the file and console.log() it.
I have been able to run xml2js by passing a small xml string to it which I also console.log() it.
But I have only been able to console.log() the file after xml2js creates the object. No matter what I try I continue to get null when using return or passing trying to pass the data outside of the initial creation of the object.
Example
This prints xml tree to the console,
var fs = require('fs');
var openFile = fs.readFile('./response.xml', 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
console.log(data);
});
function requestCreditReport() {
return openFile;
}
requestCreditReport();
This prints a small xml string to console statically added in with
xml2js,
var parseString = require('xml2js').parseString;
var xml = "<root>Hello xml2js!</root>";
parseString(xml, function (err, result) {
console.dir(result);
});
Question
How do I use the object once it is created outside of the method below. Initially when it is created I can output it to the console but cannot seem to access it outside of that console.log(). in the example below I am trying return result. This leaves the value null when I try to pass the object returned by the function to a variable like this,
var response = requestReport();
Recent try,
var fs = require('fs');
var parseString = require('xml2js').parseString;
function requestReport() {
fs.readFile('./response.xml', 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
return parseString(data, function (err, result) {
return result;
});
});
}
var response = requestReport();
console.log(response);
This returns null. But if I console.log(result) instead or using return then trying outside of the function it returns this,
{ RESPONSE_DATA:
{ '$': { MYAPI: '0.0.0' },
DETAILS: [ [Object] ],
DETAILS_ACCOUNT: [ [Object] ],
RESPONSE: [ [Object] ] } }
requestReport is async. It doesn't return anything so response is undefined.
You have to use a callback.
var fs = require('fs');
var parseString = require('xml2js').parseString;
function requestReport(callback) {
fs.readFile('./response.xml', 'utf8', function(err, data) {
if (err) return callback(err);
parseString(data, callback);
});
}
requestReport(function(err, result) {
if (err) return console.error(err);
console.log(result);
});