Why is my variable not altering after changing once - node.js

I am making a simple web-application that fetches the definition of a word using an API. There's definitely nothing wrong with the API keys because the application semi-works - Once i search for a word for the first time it will give back a result however, on the second try and onwards the definition doesn't change (The definition matches the first word instead of the latest one).
I'm thinking it has something to do with scopes and constants but I tried that already (Change var to let or const). I've read online about that callback functions are Asynchronous but I don't think that's the issue at all.
app.get('/definition/:word', (req, res) => {
if (word !== req.params.word) {
console.log("If statement");
word = req.params.word;
console.log(word);
word = word.toLowerCase();
}
options.path += '/' + language_code + '/' + word + '?fields=definitions' +
'&strictMatch=true';
url += options.path;
console.log(options.path);
request(url, options, (error, response, body) => {
if (error) {
res.redirect('/error');
} else {
let statusCode = (response.statusCode);
console.log(statusCode);
if (statusCode === 200) {
let data = JSON.parse(body);
console.log(definition);
definition = String(data.results[0].lexicalEntries[0].entries[0].senses[0].definitions);
console.log(definition);
res.render('definition', {
wordTitle: word,
definitions: definition
});
} else {
res.redirect('/error');
}
}
});
});
app.post('/', (req, res) => {
console.log("Post");
word = String(req.body.Word);
word = word.toLowerCase();
console.log(word);
res.redirect('/definition/' + word);
});
EDIT 1:
full index.js source code: https://github.com/NikodemBieniek/dictionary/blob/myfeature/server/index.js

I think the issue is with below line of code
options.path += '/' + language_code + '/' + word + '?fields=definitions' +
'&strictMatch=true';
So first time when you make API call and pass a param as 'XYZ'
the value in option.path will be
/en-gb/xyz?fields=definition&strictMatch=true
The second time you make API call and pass a param as 'PQR' the value in option.path will be
/en-gb/xyz?fields=definition&strictMatch=true/en-gb/pqr?fields=definition&strictMatch=true
Because you are doing string concatination.
The possible fix can be
app.get('/definition/:word', (req, res) => {
if (word !== req.params.word) {
console.log("If statement");
word = req.params.word;
console.log(word);
word = word.toLowerCase();
}
const options = {
host: 'https://od-api.oxforddictionaries.com',
port: '443',
path: '/api/v2/entries',
method: "GET",
headers: {
'app_id': process.env.APP_ID,
'app_key': process.env.APP_KEY,
}
};
options.path += '/' + language_code + '/' + word + '?fields=definitions' +
'&strictMatch=true';
let url = `${options.host}${options.path}`;
request(url, options, (error, response, body) => {
if (error) {
res.redirect('/error');
} else {
let statusCode = (response.statusCode);
console.log(statusCode);
if (statusCode === 200) {
let data = JSON.parse(body);
definition = String(data.results[0].lexicalEntries[0].entries[0].senses[0].definitions);
res.render('definition', {
wordTitle: word,
definitions: definition
});
} else {
res.redirect('/error');
}
}
});
});
Remove option and URL declared on line 12 and 26

Related

Requesting URL's one by one

I'm trying to get some data from a lot of URLs using 'request', but I can't manage to do it one url at a time.
I've tried to understand async/promises in order to make it work, but with no success. Trial and error didn't work.
I've seen other methods using different modules, but this requires rewriting most of my code and I believe there is an easier way to adapt it to my current code.
Here is a minimized version of the code :
const request = require('request');
const fs = require('fs');
const prod = fs.readFileSync('prod.txt', "utf8");
const prodid = prod.split("|");
var i;
var summary=[];
for (i=0;i<prodid.length;i++){
request('https://www.api.example.com/id='+prodid[i], { json: true }, (err, res, body) => {
if (err) { return console.log(err); }
if (body == 'NULL') {
console.log("Page " + i + " out of " + prodid.length + " is NULL!");
} else {
summary.push(body.items[0].Name);
summary.push(body.items[0].ISOnr);
summary.push(body.items[0].GTIN);
console.log("Page " + i + " out of " + prodid.length + " is done!");
fs.appendFileSync('data.txt',JSON.stringify(summary));
}
});
}
There is no async/promise involved in the example above, just the requests inside a loop.
From what I've seen, when I get the results, there is no particular order (probably is the order of which finishes first).
In the console, I always see page 500 out of 500, not 1/500, 2/500, etc.
What I'm trying to achieve, is by making each request in the order of URLs (preferably with a 1000ms delay between them)
You can promisify your request:
for (i = 0; i < prodid.length; i++) {
const result = await new Promise((resolve, reject) =>
request(
'https://www.api.example.com/id=' + prodid[i],
{ json: true },
(err, res, body) => {
if (err) {
reject(err);
}
if (body == 'NULL') {
console.log('Page ' + i + ' out of ' + prodid.length + ' is NULL!');
} else {
resolve(body);
}
}
)
);
if (result) {
summary.push(result.items[0].Name);
summary.push(result.items[0].ISOnr);
summary.push(result.items[0].GTIN);
console.log('Page ' + i + ' out of ' + prodid.length + ' is done!');
fs.appendFileSync('data.txt', JSON.stringify(summary));
}
}

How to consume a RESTful API in Node.js

I'm new to Node.js and I'm creating a simple pagination page. The REST API works fine, but consuming it has left me in limbo.
Here is the REST API (other parts have been taken out for brevity)
const data = req.query.pageNo;
const pageNo =
(typeof data === 'undefined' || data < 1) ? 1 : parseInt(req.query.pageNo);
let query = {};
const total = 10;
query.skip = (total * pageNo) - total;
query.limit = total;
try {
const totalCount = await Users.countDocuments();
const pageTotal = Math.ceil(totalCount / total);
const users = await Users.find({}, {}, query);
return res.status(200).json(users);
} catch (error) {
console.log('Error ', error);
return res.status(400).send(error)
};
};
When I return the json with just the 'users' object, like so return res.status(200).json(users); the page renders correctly, but when I pass in other objects like what I have in the code, it fails. This is how I'm consuming the API:
const renderHomepage = (req, res, responseBody) => {
let message = null;
if (!(responseBody instanceof Array)) {
message = 'API lookup error';
responseBody = [];
} else {
if (!responseBody.length) {
message = 'No users found nearby';
}
}
res.render('users-list', {
title: 'Home Page',
users: responseBody,
message: message
});
}
const homelist = (req, res) => {
const path = '/api/users';
const requestOptions = {
url: `${apiOptions.server}${path}`,
method: 'GET',
json: true,
};
request(
requestOptions,
(err, {statusCode}, body) => {
if (err) {
console.log('Ther was an error ', err);
} else if (statusCode === 200 && body.length) {
renderHomepage(req, res, body);
} else if (statusCode !== 200 && !body.length) {
console.log('error ',statusCode);
}
}
);
}
I've searched extensively on both here and other resources but none of the solutions quite answers my question. I hope someone could be of help

What is the Efficient way of handling text in Node.js (express framework)?

So i have a Web sever which would send HTML according to the user values.
I have a small handler which would read the existing file (contains passwords) and allow the user to enter.I't works but with some probability . i.e sometimes it would work sometimes it wont.
A snip-it which would work every time:
app.all('/acceptForm',function(req,res){
if (req.method === 'POST') {
let body = '';
var match = 0;
req.on('data', chunk => {
body += chunk.toString();
});
req.on('end', () => {
//get the uid to compare later on in the program
uid = parse(body).uid_text;
//read the UID file.
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(__dirname+'/uid.txt')
...
// write the other information to a file which would be later on re -opened again to read the things again
which have the file name of the 'uid'
firstname = parse(body).first_name;
lastname = parse(body).last_name;
mothername = parse(body).mother_name;
fathername = parse(body).father_name;
email = parse(body).email;
profession = parse(body).profession_text;
gender = parse(body).gender;
language = parse(body).lang_0;
married = parse(body).married;
birthday = parse(body).dateofbirth;
//write the UID and other things to the text file
console.log(language);
var fileContent = uid +'|' + firstname +'|'+ lastname +'|' + mothername +'|' + fathername +'|' + email+'|' + profession+'|' + gender+'|' + married+'|' +birthday + '|';
var filepath = __dirname+"/users/"+uid + ".txt";
fs.writeFile(filepath, fileContent, (err)
...
lineReader.on('line', function (line) {
if(line == uid) {
// if the uid is found...
res.cookie('name',uid, {signed: true}); //write the uid as a cookie back
res.sendFile(__dirname+'/CE/ENG/Kids.html');
} else{
//some failure message
}
});
});
}
}
The problem is as soon as the user sends this it changed to another file and the server has lost track with the client.To counteract that i added the same system with cookies.Now there is a security risk as well has many more risks.
handling a response from kids.html which is stored in another file....
(Has a very low probability that it works successfully).
app.all('/return',function(req,res){
if (req.method === 'POST') {
//read the UID file.
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(__dirname+'/uid.txt')
});
//Handling the information from the client.
lineReader.on('line', function (line) {
if(line == req.signedCookies['name']) {
//uid matches with the database
fs.readdir( __dirname+"/users/", (err, files) => {
files.forEach(file => {
if(file == req.signedCookies['name'] + ".txt"){
let questiondata = '';
req.on('data', chunk => {
questiondata += chunk.toString();
});
req.on('end', () => {
var cleaneddata = questiondata.split('%2C'); //%2C is a spliting term {array}
cleaneddata.splice(0,1);
//add the question data to another file
fs.appendFile( __dirname+"/users/" + req.signedCookies['name'] + ".txt",cleaneddata.toString() + "\r\n", function (err) { //writes inside the temp file for the questions
if (err) throw err;
fs.createReadStream( __dirname+"/users/" + req.signedCookies['name'] + ".txt").pipe(fs.createWriteStream( __dirname+'/users.txt', {flags: 'a'}));
fs.unlink( __dirname+"/users/"+ req.signedCookies['name'] + ".txt",function(err){
if(err) return console.log(err);
res.clearCookie("name");
});
});
});
}
});
})
}
As a suggestion:
var lineReader = require('linebyline');
app.post('/return' , (req, res) => {
var cookie = req.signedCookies['name'];
// check cookie
console.log(cookie);
// red the UID file:
rl = readline(__dirname + 'uid.txt');
// BTW: Why no database?!
rl.on('line', function(line, lineCount, byteCount) {
if (line === cookie) {
// ...
}
})
.on('error', function(e) {
// something went wrong
res.status(500).json({
error: err
})
});
// Here you are filling some txt files with data out of your form data..
// I highly suggest using a database instead of using your local directory
// structure and txt files.
})

how to avoid the "Allocation failed - JavaScript heap out of memory" during API import requests

I've got a nodejs script that read data in excels files and send it to a nodejs API with 'request'. this script works fine with a small amount of data. But when i test with an Excel with 60.000 lines, it breaks with error:
FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
function fillWithFile(file, path, header, fileConfig) {
let workbook = XLSX.readFile(path + '/' + file); // read file
for (var sheetIterator = 0; sheetIterator < workbook.SheetNames.length; sheetIterator++) { // for each tab in excel file
let worksheetJson = XLSX.utils.sheet_to_json(workbook.Sheets[workbook.SheetNames[sheetIterator]]); // transforme one tab of excel to JSON
let config = fileConfig[workbook.SheetNames[sheetIterator]]; // get the config for the current tab
let datasPromises = getPromises(worksheetJson, config, header);
Promise.all(datasPromises).then(function (data) {
var dataString = '{"data":' + JSON.stringify(data) + ',"options":{"purgeBefore":false}}';
let options = {
url: API_URL + '/' + config.service + 's/import',
method: 'POST',
headers: header,
body: dataString
}
request.post(options, function (err, res, body) {
if (err) throw err;
console.log('Server responded with:', body);
});
});
}
}
function getPromises(worksheetJson, config, header) {
let datasPromises = [];
let promises = [];
for (let lineIterator = 0; lineIterator < worksheetJson.length; lineIterator++) { // for each line
datasPromises.push(new Promise(function (resolve, reject) {
for (let key in config.columns) { // for each column in config
if (config.columns[key].service !== undefined) { // if service exist we need to get id of the object in this service.
promises.push(new Promise(function (resolve, reject) {
findChildren(worksheetJson[lineIterator], config.columns[key], header)
.then(function (res) {
resolve({ key: key, data: res.id });
});
}));
}
else {
promises.push(new Promise(function (resolve, reject) {
resolve({ key: key, data: worksheetJson[lineIterator][config.columns[key]] });
}));
}
}
let tmpObj = {};
Promise.all(promises).then(function (values) {
for (var i = 0; i < values.length; i++) {
tmpObj[values[i].key] = values[i].data;
}
resolve(tmpObj);
});
}));
}
return datasPromises;
}
function findChildren(sheetData, config, header) { // get children with get request
let dataObj = {};
let searchParams;
for (let key in config.columns) {
dataObj[key] = sheetData[config.columns[key]];
if (searchParams === undefined) searchParams = key + '=' + sheetData[config.columns[key]];
else searchParams += '&' + key + '=' + sheetData[config.columns[key]];
}
var headers = {
'Authorization': header.Authorization,
'Accept': 'application/json, text/plain, */*',
};
var options = {
url: API_URL + '/' + config.service + 's?' + searchParams,
headers: headers
};
return new Promise(function (resolve, reject) {
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
try {
resolve(JSON.parse(body));
} catch (e) {
reject(e);
}
}
else {
return reject(error);
}
});
});
}
The script use a huge amount of memory and he ends up to crash ...
Anyone have an idea how i can solve this ?
I think, i have to find a way to force promises to resolve instantly for avoid the memory overflow.
thanks for any help.
I think the problem is you don't control parallelism and when everything runs 'at once' - it causes memory issues.
I've tried to rewrite the code using async/await - effectively all operations are now serial:
async function fillWithFile(file, path, header, fileConfig) {
let workbook = XLSX.readFile(path + '/' + file); // read file
for (var sheetIterator = 0; sheetIterator < workbook.SheetNames.length; sheetIterator++) { // for each tab in excel file
const worksheetJson = XLSX.utils.sheet_to_json(workbook.Sheets[workbook.SheetNames[sheetIterator]]); // transforme one tab of excel to JSON
const config = fileConfig[workbook.SheetNames[sheetIterator]]; // get the config for the current tab
const data = await getData(worksheetJson, config, header);
const dataString = '{"data":' + JSON.stringify(data) + ',"options":{"purgeBefore":false}}';
const options = {
url: API_URL + '/' + config.service + 's/import',
method: 'POST',
headers: header,
body: dataString
};
await new Promise((resolve, reject) => {
request.post(options, function (err, res, body) {
if (err) {
reject(err);
return;
}
console.log('Server responded with:', body);
resolve(body);
})
});
}
}
async function getData(worksheetJson, config, header) {
const data = [];
for (let lineIterator = 0; lineIterator < worksheetJson.length; lineIterator++) { // for each line
const values = {};
for (let key in config.columns) { // for each column in config
if (config.columns[key].service !== undefined) { // if service exist we need to get id of the object in this service.
const res = await findChildren(worksheetJson[lineIterator], config.columns[key], header);
values[key] = res.id;
}
else {
values[key] = worksheetJson[lineIterator][config.columns[key]];
}
}
data.push(values);
}
return data;
}
function findChildren(sheetData, config, header) { // get children with get request
let dataObj = {};
let searchParams;
for (let key in config.columns) {
dataObj[key] = sheetData[config.columns[key]];
if (searchParams === undefined) searchParams = key + '=' + sheetData[config.columns[key]];
else searchParams += '&' + key + '=' + sheetData[config.columns[key]];
}
var headers = {
'Authorization': header.Authorization,
'Accept': 'application/json, text/plain, */*',
};
var options = {
url: API_URL + '/' + config.service + 's?' + searchParams,
headers: headers
};
return new Promise(function (resolve, reject) {
request(options, function (error, response, body) {
if (!error && response.statusCode === 200) {
try {
resolve(JSON.parse(body));
} catch (e) {
reject(e);
}
}
else {
return reject(error);
}
});
});
}
You now run import like this:
fillWithFile(..args).then(() => console.log('done'))

Why is this not working, And how do you debug code in nodejs?

First I'm trying to learn nodejs, and for that I am writing like a router. As you have in express.
This is my code:
function sirus() {
var util = utilFuncs(),
paths = {};
this.router = util.handleReq;
this.con = {
get: function (path, fn, options) {
options = (options || {}).method = "GET";
util.makeRequest(path, fn, options);
},
post: function (path, fn, options) {
options = (options || {}).method = "POST";
util.makeRequest(path, fn, options);
}
};
this.req = {
get: function (path, fn) {
util.addPath("GET", path, fn);
},
post: function (path, fn) {
util.addPath("POST", path, fn);
}
};
function utilFuncs() {
function handleReq(req, res) {
var url = parsePath(req.url);
var path = paths[req.method + url];
// console.log(req.url, url +' requested');
if (typeof path != "function") {
res.writeHead(404);
} else {
path(req, res);
}
res.end();
}
function addPath(type, path, callback) {
// add path as a key of object for easier look up, root=/, /a=/a, /a/=/a, /a?=/a, /a/?..=/a so there are no duplicates path=parsePath(path);
paths[type + path] = callback;
}
function parsePath(path) {
path = url.parse(path).pathname;
if ((/[^\/]+\/(?=$)/igm).test(path)) path = path.substring(0, path.length - 1);
return path;
}
function makeRequest(path, fn, options) {
var urls = url.parse(path);
var d = {
host: null,
hostname: null,
method: "GET",
path: '/',
port: 80,
headers: {},
auth: null,
agent: false,
keepAlive: false
};
for (var k in options) d[k] = options[k];
d.host = urls.host;
d.hostname = urls.hostname;
d.path = urls.pathname;
d.headers['Content-Type'] = 'application/x-www-form-urlencoded';
d.headers['Content-Length'] = ((d || {}).body || {}).length || '';
var req = http.request(options, function (res) {
// console.log('STATUS: ' + res.statusCode);
// console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function (chunk) {
data += chunk;
});
res.on('response', function () {
fn(res);
});
});
req.on('error', function (e) {
console.log('problem with request: ' + e.message);
});
req.write(d.body);
req.end();
}
return {
makeRequest: makeRequest,
handleReq: handleReq,
addPath: addPath
};
}
}
and i use it like this:
var http = require('http'),
url = require('url'),
app = new sirus();
http.createServer(function (req, res) {
app.router(req, res);
}).listen(80, '127.0.0.1');
app.req.get('/', function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var link = "https://www.reddit.com/";
res.write('Click here');
});
app.con.get('http://www.google.com/', function (res) {
console.log('from req: ' + res);
});
The error i get is The First argument must be string or buffer
Receiving part works correctly, but when i started to add make request part something isn't right.
Also since I'm new to JS and Nodejs. I'd like general advice or anything that catch your eye something that i can be improved about this code. Since I'm note sure i am doing things the best way.
On line 85
req.write(d.body);
there is no member named "body" in the "d" object.
1 - Why is this not working - you get exactly error that tells you why it is not working. If you check more details about error it shows the file and which line error happened at. Perhaps the arguments you pass to function call are not what Express is expecting.
2 - How to debug - look at node.js package called nodev.

Resources