Well, have no clue what am I missing.
I got a file with lines like that (time series you can say):
fs1,disk1,<date>,<value>
fs1,disk1,<date>,<value>
fs2,disk2,<date>,<value>
fs3,disk3,<date>,<value>
fs3,disk3,<date>,<value>
fs3,disk3,<date>,<value>
fs4,disk4,<date>,<value>
The lines are sorted (fs,disk,date)
so I want to read this file line by line with nodeJS readline, and as long as i have the same fs,disk to concatante this date,value series into a string and then to store that sting inside the dictionary..
for some silly reason that I miss, its not doing so.
here is the code:
var express = require('express');
var app = express();
var path = require('path');
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(__dirname + '/disksTimeSeries/timeSeries')
});
app.use(express.static('public'))
app.get('/', function(req, res) {
res.send(index.html);
});
app.listen(3005);
var diskToTimeSeries = {};
var currDisk;
var prevDisk;
var currStr = "";
var seriesStr="";
var i = 0;
var disk;
var splitArr;
var data = {};
data['prevDisk'] = "";
data['seriesStr'] = "";
lineReader.on('line', function (line) {
console.log(line);
splitArr = line.split(",");
disk = splitArr[0] + "," + splitArr[1];
if (data['prevDisk'] && disk !== data['prevDisk'])
{
diskToTimeSeries[data['prevDisk']] = data['seriesStr'];
data['seriesStr'] = "";
}
else
{
data['prevDisk'] = disk;
data['seriesStr'] = data['seriesStr'] + (splitArr[2] + "," + splitArr[3] + "\n");
}
console.log(disk);
});
// viewed at http://localhost:8080
app.get('/disks', function(req, res) {
var diskIds = req.query.ids.split(":");
var resultStr = "";
diskIds.forEach(function(diskName){
resultStr += "disk name--" + diskName + "\n" + diskToTimeSeries[diskName];
});
res.send(resultStr);
});
module.exports = app;
What am I missing :( ?
Related
There is a function defined in a route polygone.js :
var express = require('express');
var router = express.Router();
...
function transformPolygoneFromDbToText(polygoneArrayTxt, cb) {
var sommets = JSON.parse(polygoneArrayTxt) , ret = "" , separator = "";
if (polygoneArrayTxt != "") {
async.eachOf(sommets, function (sommet, position, cb_) {
var point = JSON.parse(sommet);
var longitude = point[0], latitude = point[1];
ret += separator + longitude + " " + latitude;
separator = ",";
cb_();
}, function () {
cb(ret);
});
}
else
cb("");
}
module.exports = router;
I want to call the function transformPolygoneFromDbToText from another route. How to do that ?
For the function transformPolygoneFromDbToText if you have any dependencies then move it to a separate file so that if you can reuse it anywhere you what.
create a file name utils.js
function transformPolygoneFromDbToText(polygoneArrayTxt, cb) {
var sommets = JSON.parse(polygoneArrayTxt) , ret = "" , separator = "";
if (polygoneArrayTxt != "") {
async.eachOf(sommets, function (sommet, position, cb_) {
var point = JSON.parse(sommet);
var longitude = point[0], latitude = point[1];
ret += separator + longitude + " " + latitude;
separator = ",";
cb_();
}, function () {
cb(ret);
});
}
else
cb("");
}
module.exports = { transformPolygoneFromDbToText }
and import like below in route file and export the route to use it in app.js or index.js
const { formatErrorResponse } = require('./utils');
I'm trying to write to disk a file uploaded POST inside a form with the attribute enctype="multipart/form-data", and i get the data in server and managed to sort it (get contents inside file and filename), but when i write in disk, the file is not readable. These images can illustrate my situatuion. Image1 shows how i'm getting data amd image2 shows how it is displayed. I don't know what's wrong.
I've trid encoding as base64, binary, and no encoding sending it straight as it comes from form, but no success.
Image 1 http://es.tinypic.com/view.php?pic=sou610&s=9#.XRuig-tKiUk
Image 2 http://es.tinypic.com/view.php?pic=11jag49&s=9#.XRujROtKiUk
The file gets stored in server, but it is not readable. I hope i could manage to get the file in server just as it is uploaded.
if(request_param.method == "POST")
{
var qs = require('querystring');
var body = '';
request_param.on('data', function(data)
{
body += data; //gets incoming data on buffer
});
request_param.on('end', function()
{
var util = require('util');
var postvars_array = qs.parse(body);
var boundary = body.substring(0, 40); //gets the boundary
var line_separator = "\r\n";
var array_post_fields = body.split(boundary);
var array_post_field;
var field_header_array;
var file_content;
var file_name;
var file_name_arr;
var file_ext;
var file_ext_arr;
for(f = 1; f < array_post_fields.length; f++) //parses the string from multipart form
{
array_post_field = array_post_fields[f].split(line_separator);
field_header_array = array_post_field[1].split(";");
if(field_header_array.length > 2)
{
file_name = field_header_array[2]; //gets filename
file_name_arr = file_name.split("=");
file_name = file_name_arr[1];
file_name = file_name.replace('"', '');
file_name = file_name.replace('"', '');
file_content = array_post_field[4]; //gets file contents
}
}
cont += '<br/><br/>';
cont += '<br/><br/>';
cont += file_name + '<br/><br/>' + file_content; //prints file contents, lots of diamonds (question signs)
file_ext_arr = file_name.split(".");
file_ext = file_ext_arr[file_ext_arr.length - 1];
if(file_name != "")
{
const fs = require('fs');
fs.writeFile("./shared/"+file_formats[file_ext]+"/"+file_name, file_content, function(err) //writes on disk (file gets saved but its unreadable)
{
if(err)
return console.log(err);
var fs = require('fs');
fs.readFile("./media/text/template.html", 'utf8', function(err, contents)
{
if(err)
console.log(err);
cont = contents.replace("<hydra-html-content>", cont);
cont = cont.replace("<hydra-module-name>", "Archivos");
cont = cont.replace("<hydra-title-page>", ".:Hydra / Archivos:.");
response_param.writeHead(200, {'Content-Type': 'text/html; charset=utf-8'});
response_param.write(cont);
response_param.end();
});
});
}
else
{
var fs = require('fs');
fs.readFile("./media/text/template.html", 'utf8', function(err, contents)
{
if(err)
console.log(err);
cont = contents.replace("<hydra-html-content>", cont);
cont = cont.replace("<hydra-module-name>", "Archivos");
cont = cont.replace("<hydra-title-page>", ".:Hydra / Archivos:.");
response_param.writeHead(200, {'Content-Type': 'text/html; charset=utf-8'});
response_param.write(cont);
response_param.end();
});
}
});
}
I recently stating coding in node.js and might be a very simple question.
Trying to write a XML parser/validator to validate xml schema and values against values/ xpath stored in an excel sheet.
Now once the validation function is complete I want to call a printResult function to print final result. However if I try to call the function immediately after the first function .. its printing variables initial values and if called within the first which is iterating though the number of xpaths present in excel sheet and printing result with increments.
var mocha = require('mocha');
var assert = require('chai').assert;
var fs = require('fs');
var parseString = require('xml2js').parseString;
var xpath = require('xpath');
var dom = require('xmldom').DOMParser;
var XLSX = require('xlsx');
var Excel = require("exceljs");
var should = require('chai').should();
var HashMap = require('hashmap');
var colors = require('colors');
require('/xmlValidator/dbConnect.js');
var map = new HashMap();
var elementMap = new HashMap();
var resultValue;
//console.log('hello'.green);
map.set("PASS", 0);
map.set("FAIL", 0);
map.set("INVALID_PATH", 0);
function computeResult(elementPath, result) {
var pass = map.get("PASS");
var fail = map.get("FAIL");
var invalidPath = map.get("INVALID_PATH");
elementMap.set(elementPath, result);
if (result == "PASS") {
pass++;
map.set("PASS", pass);
} else if (result == "FAIL") {
fail++;
map.set("FAIL", fail);
} else {
invalidPath++;
map.set("INVALID_PATH", invalidPath)
}
printResult();
}
function printResult() {
var pass = map.get("PASS");
var fail = map.get("FAIL");
var invalidPath = map.get("INVALID_PATH");
console.log(("PASS Count :" + pass).green);
console.log(("FAIL Count :" + fail).red);
console.log(("Inavlid Path :" + invalidPath).yellow);
elementMap.forEach(function(value, key) {
if (value == "INVALID_PATH")
console.log((key + ":" + value).yellow);
else if (value == "FAIL")
console.log((key + ":" + value).red);
else
console.log(key + ":" + value);
});
}
var workbook = new Excel.Workbook();
workbook.xlsx.readFile('utils/' + process.argv[2])
.then(function() {
var worksheet = workbook.getWorksheet(1);
worksheet.eachRow(function(row, rowNumber) {
//console.log(rowNumber);
var row = worksheet.getRow(rowNumber);
var dataPath1 = row.getCell("A").value;
var dataPath2 = row.getCell("B").value;
var dataPath = dataPath1 + dataPath2;
//console.log(dataPath);
var dataValue = row.getCell("D").value;
var flag = row.getCell("E").value;
//console.log(flag)
//console.log(dataValue);
if (!flag)
validate(dataPath, dataValue, rowNumber);
//else console.log("NOT EXECUTED" + rowNumber)
});
})
function validate(dataPath, dataValue, rowNumber) {
var fail = 0;
fs.readFile('utils/' + process.argv[3], 'utf8', function(err, data) {
if (err) {
console.log("ERROR ERROR ERROR ERROR ");
return console.log(err);
}
var doc = new dom().parseFromString(data);
var subId = String(xpath.select1(dataPath, doc));
if (subId == "undefined") {
/*console.log('undefined caught');
console.log("row number :" + rowNumber);*/
var resultValue = "INVALID_PATH";
computeResult(dataPath, resultValue);
} else {
var subId = xpath.select1(dataPath, doc);
var value = subId.lastChild.data;
/*console.log("row number :" + rowNumber);
console.log("actual value: " + value);
console.log("expected value:" + dataValue );*/
if (dataValue == null) {
assert.notEqual(value, dataValue, "value not found");
resultValue = "PASS";
computeResult(dataPath, resultValue);
} else {
if (value == dataValue)
resultValue = "PASS";
else resultValue = "FAIL";
computeResult(dataPath, resultValue);
}
}
});
}
In the code above i want to call printResult() function after validate function is completely executed (workbook.xlsx.readFile)
Can some one please help me out how to use done () function or make sync call ?
If fs.readFileAsync('utils/' + process.argv[3], 'utf8') can be executed once, then validate() will be completely synchronous and calling printResult() after the verification loop will be trivial.
In the main routine, you can simply aggregate two promises ...
var promise1 = workbook.xlsx.readFile();
var promise2 = fs.readFileAsync(); // requires `fs` to be promisified.
... before embarking on the verification loop.
Promise.all([promise1, promise2]).spread(/* verify here */);
Also a whole bunch of tidying can be considered, in particular :
establishing "PASS", "FAIL" and "INVALID_PATH" as constants to avoid lots of repetitive string creation,
using js plain objects in lieu of hashmap,
building map from elementMap inside the print function.
having validate() return its result and building elementMap in the main routine
Here's the whole thing, about as tight as I can get it. I may have made a few assumptions but hopefully not too many bad ones ...
var mocha = require('mocha');
var assert = require('chai').assert;
var Promise = require("bluebird");
var fs = Promise.promisifyAll(require("fs")); // allow Bluebird to take the pain out of promisification.
var parseString = require('xml2js').parseString;
var xpath = require('xpath');
var dom = require('xmldom').DOMParser;
var XLSX = require('xlsx');
var Excel = require("exceljs");
var should = require('chai').should();
// var HashMap = require('hashmap');
var colors = require('colors');
require('/xmlValidator/dbConnect.js');
const PASS = "PASS";
const FAIL = "FAIL";
const INVALID_PATH = "INVALID_PATH";
function printResult(elementMap) {
var key, result,
map = { PASS: 0, FAIL: 0, INVALID_PATH: 0 },
colorNames = { PASS: 'black', FAIL: 'red', INVALID_PATH: 'yellow' };
for(key in elementMap) {
result = elementMap[key];
map[(result === PASS || result === FAIL) ? result : INVALID_PATH] += 1;
console.log((key + ": " + result)[colorNames[result] || 'black']); // presumably colors can be applied with associative syntax? If so, then the code can be very concise.
}
console.log(("PASS Count: " + map.PASS)[colorNames.PASS]);
console.log(("FAIL Count: " + map.FAIL)[colorNames.FAIL]);
console.log(("Inavlid Path: " + map.INVALID_PATH)[colorNames.INVALID_PATH]);
}
function validate(doc, dataPath, dataValue) {
var subId = xpath.select1(dataPath, doc),
value = subId.lastChild.data,
result;
if (String(subId) == "undefined") {
result = INVALID_PATH;
} else {
if (dataValue === null) {
assert.notEqual(value, dataValue, "value not found"); // not too sure what this does
result = PASS;
} else {
result = (value === dataValue) ? PASS : FAIL;
}
}
return result;
}
//Main routine
var workbook = new Excel.Workbook();
var promise1 = workbook.xlsx.readFile('utils/' + process.argv[2]); // from the question, workbook.xlsx.readFile() appears to return a promise.
var promise2 = fs.readFileAsync('utils/' + process.argv[3], 'utf8');
Promise.all([promise1, promise2]).spread(function(data2, data3) {
var worksheet = workbook.getWorksheet(1),
doc = new dom().parseFromString(data3),
elementMap = {};
worksheet.eachRow(function(row, rowNumber) {
// var row = worksheet.getRow(rowNumber); // row is already a formal variable ???
var dataPath, dataValue;
if (!row.getCell('E').value)
dataPath = row.getCell('A').value + row.getCell('B').value;
dataValue = row.getCell('D').value;
elementMap[dataPath] = validate(doc, dataPath, dataValue);
});
printResult(elementMap);
});
Untested so may not run but at least you can raid the code for ideas.
How do we upload an image received form the mobile app to another server using the request module in nodejs?
I have tried using the multipart module to extract the file and send it in formData attribute with the request module (post method). This doesn't seem to work.
Please Use following code, it has been tested in express. you can modify it according to your requirement
var path = require('path');
var util = require('util');
if (req.files.profile_image !== undefined) {
var file = req.files.profile_image;
var tmp_path = file.path;
var fileName = file.name;
var milliseconds = new Date().getTime();
var file_ext = fileName.substr((Math.max(0, fileName.lastIndexOf(".")) || Infinity) + 1);
var newFileName = requestedUser + '_' + milliseconds + '.' + file_ext;
var pathToFile = require('path').dirname(require.main.filename);
var mainPath = path.dirname(pathToFile)
var target_path = path.join(mainPath, 'public/uploads/users', newFileName);
var readStream = fs.createReadStream(tmp_path)
var writeStream = fs.createWriteStream(target_path);
util.pump(readStream, writeStream, function(err) {
if (err) {
//handle error
} else {
//successfully uploaded
}
});
} else {
//file not recieved
}
Thanks
I could use a hand. My company uses Google Sheets extensively, and we need a way to access files when we lose our Internet connection.
I could not get any of the examples found on this site to work for creating xls or ods from Google Sheets via script.
I did script a way to create csv backups, accessible from a local Google Drive folder. When used with an hourly trigger, this script creates csv files of every sheet of any spreadsheet modified in the last hour, puts them in a folder, and zips it inside a folder specifically for backups. From there I can move the zip to our local server.
function backUpMaker() {
var backupFolderId = '<Id of Backup Folder>';
var timeNow = new Date();
var newFolder = DocsList.createFolder(timeNow);
var newFolderId =newFolder.getId();
newFolder.addToFolder(DocsList.getFolderById(backupFolderId));
newFolder.removeFromFolder(DocsList.getRootFolder());
var sheets = DriveApp.getFilesByType(MimeType.GOOGLE_SHEETS);
while (sheets.hasNext()) {
var sheet = sheets.next();
var lastUpdate = sheet.getLastUpdated();
var timeHourAgo = new Date();
timeHourAgo = timeNow - 3600000;
if(lastUpdate >= timeHourAgo){
var sheetId = sheet.getId();
var csv = eachSheet(sheetId,newFolderId);
}
}
var backupFolder = DocsList.getFolderById(backupFolderId);
try{
backupFolder.createFile(Utilities.zip(newFolder.getFiles(), timeNow + '.zip'));
} catch(err) {
Logger.log(err);
}
newFolder.setTrashed(true);
}
function eachSheet(key,newFolderId) {
var ss = SpreadsheetApp.openById(key);
var ssId = ss.getId();
var ssName = ss.getName();
var howManySheets = ss.getNumSheets();
try{
for (var sheetIndex=0; sheetIndex < howManySheets; sheetIndex++) {
var activeSheet = ss.getSheets()[sheetIndex];
var activeName = activeSheet.getName();
if(activeName != 'Dropdowns'){ //Skip a hidden sheet used for validation on many of our spreadsheets
var activeId = activeSheet.getSheetId();
var time = new Date();
var fileName = time + " Backup: " + ssName + " " + activeName + ".csv";
var csv = contentCSV(ssId,activeId);
var folder = DocsList.getFolderById(newFolderId);
folder.createFile(fileName, csv, 'text/plain');
}
}
} catch(err) {
Logger.log(err)
}
}
function contentCSV(key,gid) {
var file = DocsList.getFileById(key);
var response = UrlFetchApp.fetch("https://spreadsheets.google.com/feeds/download/spreadsheets/Export?key=" + key +
"&gid=" + gid + "&exportFormat=csv", oAuth());
var fileText = response.getContentText();
return fileText;
}
function oAuth() {
var oauthConfig = UrlFetchApp.addOAuthService("spreadsheets");
var scope = "https://spreadsheets.google.com/feeds"
oauthConfig.setConsumerKey("anonymous");
oauthConfig.setConsumerSecret("anonymous");
oauthConfig.setRequestTokenUrl("https://www.google.com/accounts/OAuthGetRequestToken?scope="+scope);
oauthConfig.setAuthorizationUrl("https://accounts.google.com/OAuthAuthorizeToken");
oauthConfig.setAccessTokenUrl("https://www.google.com/accounts/OAuthGetAccessToken");
var requestData = {
"oAuthServiceName": "spreadsheets",
"oAuthUseToken": "always",
};
return requestData;
}
How would I modify this to save as ods rather than csv? Or is there a better way to save backups? Thank you for your help!
Working code follows, thanks to input by #serge-insas :
Note: To make this work, enter your backup folder's id key, then choose ods or xlsx by commenting out the other choice. Then you need to set up a timed hourly trigger for backUpMaker(). Logs will catch errors for files that have been moved temporarily. Here goes:
function backUpMaker() {
var backupFolderId = '0B5--------------------1ZX1k';
//var exportFormat = 'ods';
var exportFormat = 'xlsx';
var timeNow = new Date();
var newFolder = DocsList.createFolder(timeNow);
var newFolderId =newFolder.getId();
newFolder.addToFolder(DocsList.getFolderById(backupFolderId));
newFolder.removeFromFolder(DocsList.getRootFolder());
var sheets = DriveApp.getFilesByType(MimeType.GOOGLE_SHEETS);
while (sheets.hasNext()) {
var sheet = sheets.next();
var lastUpdate = sheet.getLastUpdated();
var timeHourAgo = new Date();
timeHourAgo = timeNow - 3600000;
if(lastUpdate >= timeHourAgo){
try{
var key = sheet.getId();
var name = sheet.getName();
var fileName = timeNow + " Backup: " + name + "." + exportFormat;
var blob = contentBackup(key,exportFormat);
var folder = DocsList.getFolderById(newFolderId);
folder.createFile(blob).rename(fileName);
} catch(err){
Logger.log(err);
}
}
}
var backupFolder = DocsList.getFolderById(backupFolderId);
try{
backupFolder.createFile(Utilities.zip(newFolder.getFiles(), timeNow + '.zip'));
} catch(err) {
Logger.log(err);
}
newFolder.setTrashed(true);
}
function contentBackup(key,exportFormat) {
try{
if(exportFormat == 'xlsx'){
exportFormat = 'xls';
}
var file = DocsList.getFileById(key);
var response = UrlFetchApp.fetch("https://spreadsheets.google.com/feeds/download/spreadsheets/Export?key=" + key +
"&exportFormat=" + exportFormat, oAuth());
var fileBlob = response.getBlob();
return fileBlob;
} catch(err) {
Logger.log(err)
}
}
function oAuth() {
var oauthConfig = UrlFetchApp.addOAuthService("spreadsheets");
var scope = "https://spreadsheets.google.com/feeds"
oauthConfig.setConsumerKey("anonymous");
oauthConfig.setConsumerSecret("anonymous");
oauthConfig.setRequestTokenUrl("https://www.google.com/accounts/OAuthGetRequestToken?scope="+scope);
oauthConfig.setAuthorizationUrl("https://accounts.google.com/OAuthAuthorizeToken");
oauthConfig.setAccessTokenUrl("https://www.google.com/accounts/OAuthGetAccessToken");
var requestData = {
"oAuthServiceName": "spreadsheets",
"oAuthUseToken": "always",
};
return requestData;
}
Your code needed very few changes to get ods files.
2 differences : the requested format in the urlFetch becomes "ods" and the returned object is not a string anymore but a blob.
Beside that, I kept all the sheet selection functions that you wrote and it seems to work fine as well. (nice job you did !).
Below is the full code I reproduced from yours and changed as described. My tests were working fine.
function backUpMaker() {
var backupFolderId = '0B3qS--------------TXdlY2M';
var timeNow = new Date();
var newFolder = DocsList.createFolder(timeNow);
var newFolderId =newFolder.getId();
newFolder.addToFolder(DocsList.getFolderById(backupFolderId));
newFolder.removeFromFolder(DocsList.getRootFolder());
var sheets = DriveApp.getFilesByType(MimeType.GOOGLE_SHEETS);
while (sheets.hasNext()) {
var sheet = sheets.next();
var lastUpdate = sheet.getLastUpdated();
var timeHourAgo = new Date();
timeHourAgo = timeNow - 3600000;
if(lastUpdate >= timeHourAgo){
var sheetId = sheet.getId();
var csv = eachSheet(sheetId,newFolderId);
}
}
var backupFolder = DocsList.getFolderById(backupFolderId);
try{
backupFolder.createFile(Utilities.zip(newFolder.getFiles(), timeNow + '.zip'));
} catch(err) {
Logger.log(err);
}
newFolder.setTrashed(true);
}
function eachSheet(key,newFolderId) {
var ss = SpreadsheetApp.openById(key);
var ssId = ss.getId();
var ssName = ss.getName();
var howManySheets = ss.getNumSheets();
try{
for (var sheetIndex=0; sheetIndex < howManySheets; sheetIndex++) {
var activeSheet = ss.getSheets()[sheetIndex];
var activeName = activeSheet.getName();
if(activeName != 'Dropdowns'){ //Skip a hidden sheet used for validation on many of our spreadsheets
var activeId = activeSheet.getSheetId();
var time = new Date();
var fileName = time + " Backup: " + ssName + " " + activeName + ".ods";
var blob = contentODS(ssId,activeId);
var folder = DocsList.getFolderById(newFolderId);
folder.createFile(blob).rename(fileName);
}
}
} catch(err) {
Logger.log(err)
}
}
function contentODS(key,gid) {
var file = DocsList.getFileById(key);
var response = UrlFetchApp.fetch("https://spreadsheets.google.com/feeds/download/spreadsheets/Export?key=" + key +
"&gid=" + gid + "&exportFormat=ods", oAuth());
var fileBlob = response.getBlob();
return fileBlob;
}
function oAuth() {
var oauthConfig = UrlFetchApp.addOAuthService("spreadsheets");
var scope = "https://spreadsheets.google.com/feeds"
oauthConfig.setConsumerKey("anonymous");
oauthConfig.setConsumerSecret("anonymous");
oauthConfig.setRequestTokenUrl("https://www.google.com/accounts/OAuthGetRequestToken?scope="+scope);
oauthConfig.setAuthorizationUrl("https://accounts.google.com/OAuthAuthorizeToken");
oauthConfig.setAccessTokenUrl("https://www.google.com/accounts/OAuthGetAccessToken");
var requestData = {
"oAuthServiceName": "spreadsheets",
"oAuthUseToken": "always",
};
return requestData;
}