How to convert properties file to xml node.js - node.js

I have properties File and I want to convert it to XML by using node.js.I don't know how can I convert, please help me

const builder = require('xmlbuilder');
const fs = require('fs');
const path = require('path');
const PROPS_FILE = //PROPS_FILE_PATH;
const XML_FILE = //XML_FILE_PATH;
let xml = builder.create('resources');
let readFromPropsFile = function(){
let promise = new Promise(function(resolve, reject){
console.log(PROPS_FILE);
const readline = require('readline');
const rl = readline.createInterface({
input: fs.createReadStream(path.resolve(__dirname,PROPS_FILE))
});
rl.on('line', (line) => {
let keyValuePair = line.split('=');
// converting key valur to XML
convertToXML(keyValuePair[0], keyValuePair[1]);
});
rl.on('error', function(e) {
console.log(e);
reject(e);
// something went wrong
});
rl.on('close', function(line){
console.log(`done reading file ${PROPS_FILE}.`);
resolve();
})
});
return promise;
};
let convertToXML= function(key, value){
xml.ele('string', {'name': `${key}`}, `${value}`);
};
let writeInToXMLFile = function(){
let promise = new Promise(function(resolve, reject){
fs.writeFile(path.resolve(__dirname,XML_FILE) , xml, function (error) {
if (error) {
console.error(error);
reject(new Error(error));
} else {
console.log(`file ${XML_FILE} saved `);
resolve();
}
});
});
return promise;
}
readFromPropsFile()
.then(writeInToXMLFile)
.catch((err)=>{ console.log(err) });

Related

Return result from python script in Node JS child process

Im trying to return the value from a python script from a nodejs child process and i just cant seem to get it to work, it prints in the console using console.log correctly as it should but only returns undefined, i was wondering if there is a way to directly return that value, or to parse the console.log results into a string.
var sys = require('util');
module.exports = function() {
this.getPlay = function getPlaylist(name) {
const childPython = spawn('python' ,['main.py', name]);
var result = '';
childPython.stdout.on(`data` , (data) => {
result += data.toString();
});
childPython.on('exit' , () => {
console.log(result);
});
}};
Python script is empty for now and prints "Hello 'name' "
Edit:
I tried to use promises and here is what i have:
(async function(){
function test(name) {
return new Promise((resolve , reject) => {
const childPython = spawn('python' ,['main.py', "He"]);
var result = '';
childPython.stdout.on(`data` , (data) => {
result += data.toString();
});
childPython.on('close' , function(code) {
t = result
resolve(result)
});
childPython.on('error' , function(err){
reject(err)
});
})};
var t;
await test(name);
console.log(t);
return t;
})();
Define it like this.
function getPlaylist(name) {
return new Promise((resolve , reject) => {
const childPython = spawn('python' ,['main.py', name]);
var result = '';
childPython.stdout.on(`data` , (data) => {
result += data.toString();
});
childPython.on('close' , function(code) {
resolve(result)
});
childPython.on('error' , function(err){
reject(err)
});
})
};
Remeber to use try...catch for it it gets rejected. https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/try...catch
async function runTest() {
try {
const playList = await getPlaylist();
console.log(playList);
} catch (err) {
}
}
runTest()
const {spawn} = require('child_process');
const getPythonScriptStdout = (pythonScriptPath) => {
const python = spawn('python', [pythonScriptPath]);
return new Promise((resolve, reject) => {
let result = ""
python.stdout.on('data', (data) => {
result += data
});
python.on('close', () => {
resolve(result)
});
python.on('error', (err) => {
reject(err)
});
})
}
getPythonScriptStdout('./python.py').then((output) => {
console.log(output)
})
python.py file
print("hi from python")

Export CSV without crash from node js

I need to export CSV for large amount of data, like it's having 100k rows, I'm using Json2CSV but sometimes it's taking a long time and crashed. I'm using node js stream but no use. I am googling but I don't have idea to clear it.Please any one can guide me how can I fix it.
Node JS
var formatedData = {......} //object with data
let filename = 'test' + '.csv';
let pathName = await writeToCSV(filename, formatedData, fields);
let readStream = fs.createReadStream(pathName);
res.setHeader('Content-disposition', `attachment; filename=${filename}`);
res.set('Content-Type', 'text/csv');
let downloadStream = readStream.pipe(res);
fields = null;
formatedData = null;
downloadStream.on('finish', function() {
fs.unlink(pathName, function() {});
downloadStream = null;
readStream = null;
});
writeToCSV
function writeToCSV(filename, data, fields, option) {
return new Promise((resolve, reject) => {
if (typeof data !== 'object') {
return reject(new Error('Data is not an object'));
}
let options = {
fields
};
if (typeof option === 'object') {
for (let key in option) {
options[key] = option[key];
}
}
let tmpPath = path.join(__dirname, '..', tmp);
let pathFile = tmpPath + filename;
return Promise.all([Promise.resolve(json2csv(data, options).split('\n')), checkTMPExist(tmpPath)]).then(data => {
let csvFormat = data[0];
let writeStream = fs.createWriteStream(pathFile);
csvFormat.forEach((lines, index) => {
if (index === csvFormat.length - 1) {
writeStream.end(lines + '\n');
} else {
writeStream.write(lines + '\n');
}
});
writeStream.on('finish', function() {
this.end();
return resolve(pathFile);
});
writeStream.on('error', function(err) {
this.end();
fs.unlink(pathFile, () => {});
return reject(err);
});
}).catch(err => {
fs.unlink(pathFile, () => {});
return reject(err);
});
});
}
Front end Ajax call
function export_csv(url, fileName) {
$.ajax({
url: url,
type: "GET",
success: function (result) {
var encodedUri = 'data:application/csv;charset=utf-8,' + encodeURIComponent(result);
var link = document.createElement("a");
link.setAttribute("href", encodedUri);
link.setAttribute("download", fileName);
document.body.appendChild(link);
link.click();
},
error: function (xhr) {
console.log("Export csv have some issue:" + JSON.stringify(xhr));
}
});
}

Node.js hangs on HTTP get when download files

I want to download a series of ts files, but every time I run this code, it will be stuck in the download process of a file without any error message.
However, if you put the url in the program in the address bar of the browser, or use a tool like wget, you can still download it.
const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');
async function doRequest(url) {
return new Promise(function (resolve) {
var req = https.get(url, function (response) {
resolve(response);
});
req.end('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
req.on('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
});
}
(async() => {
for(let i = 21; i<100; i++)
{
var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';
await console.log('downloading '+i);
var file_ts_path = fs.createWriteStream(i+'.ts');
let responseRes = await doRequest(ts_url);
await responseRes.pipe(file_ts_path);
}
})();
const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');
async function doRequest(url) {
return new Promise(function (resolve) {
var req = https.get(url, function (response) {
resolve(response);
});
req.end('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
req.on('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
});
}
(async() => {
for(let i = 21; i<100; i++)
{
var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';
await console.log('downloading '+i);
var file_ts_path = fs.createWriteStream(i+'.ts');
let responseRes = await doRequest(ts_url);
await responseRes.pipe(file_ts_path);
}
})();
const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');
async function doRequest(url) {
return new Promise(function (resolve) {
var req = https.get(url, function (response) {
resolve(response);
});
req.end('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
req.on('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
});
}
(async() => {
for(let i = 21; i<100; i++)
{
var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MmFmN2JkNzEtODc3Mi00OThmLThjNzQtN2Y5NTQ3ZTgxYjA2IiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzEzNzU4fX19XX0_&Signature=hFOZlf5Iqf7k-LqVms1FT4LKoEM8psMVkCsdjoHGhKztNTzROyGgfHg34RYr3ezffQaQV6drXBuID1NMypQhwSXgJ-ZRAoGC3KnKtrfm9bSpK2Wq97sZf97D5PbBn7wNaJhmfvbztym-cRknztepOqM2v~KvCz6~esS99TOsWbtQFBZWLPacp5MV3v5eZ4wh2WJXX1jDqI1XmpZ0jyU2oJCXOgVbvU1aF86E7duvniDrbhmS1R00~tWTAETBbmBSubDw-7fGq7XzeZcFRfXbdwb0a9KwsAGh54lj1UBUMsDzEtH8vI8r9aC~MnFIRub1KxsFSOzUlRLYRp4GsPJQiQ__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=2af7bd71-8772-498f-8c74-7f9547e81b06';
await console.log('downloading '+i);
var file_ts_path = fs.createWriteStream(i+'.ts');
let responseRes = await doRequest(ts_url);
await responseRes.pipe(file_ts_path);
}
})();
You need to download a file, write it to stream, and then pipe it.
https://nodejs.org/api/https.html#https_https_get_url_options_callback
https://nodejs.org/api/fs.html#fs_fs_createwritestream_path_options
var https = require('https');
var fs = require('fs');
function doRequest(url, dest) {
return new Promise(function (resolve) {
https.get(url, res => {
let data = '';
res.on('data', (buffer) => {
data += buffer; // save downloaded data
});
res.on('end', () => {
var stream = fs.createWriteStream(dest);
stream.write(data, 'binary');
stream.on('finish', () => {
resolve('File Saved !');
});
res.pipe(stream);
});
}).on('error', function (e) {
reject('re try to get ts ...' + e);
});
});
}
for (let i = 21; i < 100; i++) {
var ts_url =
'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000' +
i +
'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';
doRequest(ts_url, i + '.ts')
.then(console.log)
.catch(console.log);
}

How to merge two csv files rows in node js

I have 2 csv files which have different different data but having a same header
eg. FILE 1 data is
"CODE","NAME","SUB_USER","SCORE"
"01","TEST","1","5"
"01","TEST","2","6"
other file FILE2 have data like this
"CODE","NAME","SUB_USER","SCORE"
"02","TEST2","3","5"
"02","TEST2","4","6"
so i want to merge both file create FILE3 output like this
"CODE","NAME","SUB_USER","SCORE"
"01","TEST","1","5"
"01","TEST","2","6"
"02","TEST2","3","5"
"02","TEST2","4","6"
I have tried below code
var express = require('express');
var router = express.Router();
var fs = require('fs');
var parse = require('csv-parse');
var async = require('async');
var csv = require("fast-csv");
var file1 = appRoot + '\\csvFiles\\details1.csv';
var file2 = appRoot + '\\csvFiles\\details2.csv';
var stream = fs.createReadStream(file1);
var stream2 = fs.createReadStream(file2);
var fileData1 = [],
fileData2 = [];
csv
.fromStream(stream)
.on("data", function(data) {
fileData1.push(data);
})
.on("end", function() {
console.log("done");
});
csv
.fromStream(stream2)
.on("data", function(data) {
fileData2.push(data);
})
.on("end", function() {
console.log("done");
});
var fileData3 = fileData1.concat(fileData2);
csv.writeToPath("outputfile.csv", fileData3).on("finish", function() {
console.log("END");
});
But not working don't know why?? Please help me
///**********************************************************************//
Thax for help but i got new problem here
After some changes above code start working
var file1 = appRoot + '\\csvFiles\\details1.csv';
var file2 = appRoot + '\\csvFiles\\idetails2.csv';
var stream = fs.createReadStream(file1);
var stream2 = fs.createReadStream(file2);
var fileData1 = [],
fileData2 = [],
i = 0;
csv.fromStream(stream).on("data", function(data) {
fileData1.push(data);
}).on("end", function() {
csv.fromStream(stream2).on("data", function(data) {
if (i != 0) {
fileData2.push(data);
}
i++;
}).on("end", function() {
console.log("done");
var fileData3 = fileData1.concat(fileData2);
csv.writeToPath("outputfile.csv", fileData3).on("finish", function() {
res.send('Done merge');
});
});
});
But problem is that what if my number of file increase then how i will handle that thing
The biggest problem here is a quite common one. You do async tasks but you don't wait for them to finish before you are using their result.
You concat the file data before the "end" callback for each tasks was called.
The solution is to wait for every callback to be called and THEN working with the data.
I created a small example using Promises
const file1 = 'one.csv';
const file2 = 'two.csv';
const stream = fs.createReadStream(file1);
const stream2 = fs.createReadStream(file2);
const fileData1 = [];
const fileData2 = [];
const file1Promise = new Promise((resolve) => {
csv
.parseFile(file1, {headers: true})
.on('data', function(data) {
fileData1.push(data);
})
.on('end', function() {
console.log('done');
resolve();
});
});
const file2Promise = new Promise((resolve) => {
csv
.parseFile(file2, {headers: true})
.on('data', function(data) {
fileData2.push(data);
})
.on('end', function() {
console.log('done');
resolve();
});
});
Promise.all([
file1Promise,
file2Promise,
])
.then(() => {
const fileData3 = fileData1.concat(fileData2);
console.log(fileData3);
const csvStream = csv.format({headers: true});
const writableStream = fs.createWriteStream('outputfile.csv');
writableStream.on('finish', function() {
console.log('DONE!');
});
csvStream.pipe(writableStream);
fileData3.forEach((data) => {
csvStream.write(data);
});
csvStream.end();
});
I created a function with which you can easily merge multiple files:
function concatCSVAndOutput(csvFilePaths, outputFilePath) {
const promises = csvFilePaths.map((path) => {
return new Promise((resolve) => {
const dataArray = [];
return csv
.parseFile(path, {headers: true})
.on('data', function(data) {
dataArray.push(data);
})
.on('end', function() {
resolve(dataArray);
});
});
});
return Promise.all(promises)
.then((results) => {
const csvStream = csv.format({headers: true});
const writableStream = fs.createWriteStream(outputFilePath);
writableStream.on('finish', function() {
console.log('DONE!');
});
csvStream.pipe(writableStream);
results.forEach((result) => {
result.forEach((data) => {
csvStream.write(data);
});
});
csvStream.end();
});
}
example usage
concatCSVAndOutput(['one.csv', 'two.csv'], 'outputfile.csv')
.then(() => ...doStuff);

Unable to get callback from function

Trying to get the variable random_song from the function functions.random_song
Function:
functions.random_song = () => {
fs.readFile('auto_playlist.txt', 'utf8', function(err, data) {
if (err) {
console.log(`${`ERR`.red} || ${`Error fetching song from auto playlist ${err}`.red}`);
}
let songs = data.split('\n');
songs.splice(-1, 1);
var random = Math.floor(Math.random()*songs.length);
let random_song = (songs[random]);
return random_song;
})
}
Attempting to callback the random song
functions.random_song(song => {
console.log(song)
})
The code just return undefined Ideas?
Your function:
functions.random_song = (callback) => {
fs.readFile('auto_playlist.txt', 'utf8', function(err, data) {
if (err) {
// Do stuff
return callback(err);
}
// Do stuff
callback(null, random_song);
})
}
And then call it:
functions.random_song((err, song) => {
// Do stuff
});
You may want to read more about using Promise/async-await instead of callback.
functions.random_song = () => {
let data = fs.readFileSync('auto_playlist.txt', 'utf8');
let songs = data.split('\n');
songs.splice(-1, 1);
var random = Math.floor(Math.random()*songs.length);
let rand_song = (songs[random]);
return rand_song;
}
console.log(functions.random_song())
got it working, thanks for the help <3
Following is the code which use Promise to handle file read and data processing task asynchronously
const fs = require('fs')
const functions = {}
functions.random_song = () => {
return new Promise((resolve, reject) => {
fs.readFile('auto_playlist.txt', 'utf8', function(err, data) {
if (err) {
console.log(err)
reject(err)
}
let songs = data.split('\n');
songs.splice(-1, 1);
var random = Math.floor(Math.random()*songs.length);
let random_song = (songs[random]);
resolve(random_song);
})
})
}
functions.random_song()
.then(song => console.log('Song Name', song))
.catch(err => console.error('Error fetching songs', err))
console.log('It will be executed before promise is resolved')
fs.readFile is asynchronus and your function will return before the file is read. Use fs.readFileSync instead. Like this:
functions.random_song = () => {
const data = fs.readFileSync('auto_playlist.txt', 'utf8');
let songs = data.split('\n');
songs.splice(-1, 1);
var random = Math.floor(Math.random()*songs.length);
let random_song = (songs[random]);
return random_song;
}

Resources