How do I cleanse the data (removing parenthesis and quotes) in NodeJS? - node.js

I have a csv sheet where some of the cells could have either single record or multiple records like an array. A sample cell would look like either ["London"] or ["Cairo", "Montreal", "Paris"] - with the parenthesis included. I'm using sever.js to show these records in a visualization on a webpage.
Everything's fine except the parenthesis and quotes are being shown in the visualization. How do I write a rule/logic in nodeJS such that both parenthesis and quotes are not shown in the final visualization?
There's no point in cleansing the data in the csv file itself because I'm gonna have to use this code for hundreds of csv files, so it's better to write a rule in nodeJS.
The following's the server.js code I've used so far:
/**
* API for getting country Names
*/
app.get('/get-country-names.json', (req, res) => {
william.william().then(response => {
res.json(response);
}).catch(message => res.send("Failed to read file" + message));
});
/**
* API to read file and return response in json
* Takes country id as parameter
*/
app.post('/get-city-sources.json', (req, res) => {
let countryName = req.body.countryName;
let city = req.body.city;
william.william().then(sources => {
let filteredSources = [{ name: city, parent: null, color: 'red' }];
Array.from(new Set(sources.filter(source => source.countryName === countryName && source.city === city).map(src => src.source)))
.forEach(source => {
let sourceArr = source.split(',');
console.log(sourceArr);
sourceArr.forEach(src => filteredSources.push({ name: src, parent: city, color: 'blue' }));
});
res.json(filteredSources);
}).catch(message => res.send("Failed to read file" + message));
});
app.listen(port, () => console.log(`Example app listening at http://localhost:${port}`))

Regular expression should do the trick. I'm not exactly sure where the stripping should happen within your code but this is a sample function that uses regular expression to strip out square brackets and quotes:
const formatCellString = (cell = '') => {
return cell
// strip out the square brackets
.replace(/^\[(.*?)\]$/, "$1")
// strip out the quotes
.replace(/("\b)|(\b")/g, '')
}
formatCellString('["Cairo", "Montreal", "Paris"]'); => // Cairo, Montreal, Paris
formatCellString('["London"]'); => // London

Related

Calculate sum of values in Cypress

I am using BDD/Cucumber with Cypress. I want to calculate the sum of some rows of table.
This is my step definition:
And("I add up all the total on hand",()=>{
const sumOnHand = itemListPage.totalOnHandAmsterdam()+itemListPage.totalOnHandDelft()
cy.log(sumOnHand)
})
And this is my js page:
totalOnHandAmsterdam() {
cy.get(':nth-child(2) > .dx-grandtotal > span').invoke('text').then(text =>{
const ttOnHandAmst = text
return ttOnHandAmst;
})
}
totalOnHandDelft() {
cy.get(':nth-child(11) > .dx-grandtotal > span').invoke('text').then(text =>{
const ttOnHandDelft = text
return ttOnHandDelft;
})
}
But this is the output of the calculation:
Any ideas on how can I sum up this value is appreciated.
You can't use the results of totalOnHandAmsterdam() and totalOnHandDelft() directly in a summation because
they don't return anything (the return inside .then(text => does not return the value from the function).
Cypress commands don't return values, they add the values to the command queue
You can do it like this
totalOnHandAmsterdam() {
return cy.get(':nth-child(2) > .dx-grandtotal > span')
.invoke('text').then(parseInt)
}
totalOnHandDelft() {
return cy.get(':nth-child(11) > .dx-grandtotal > span')
.invoke('text').then(parseInt)
}
And("I add up all the total on hand", () => {
itemListPage.totalOnHandAmsterdam().then(ams => // get value on command queue
itemListPage.totalOnHandDelft().then(delft => // get other value
const sumOnHand = ams + delft;
cy.log(sumOnHand)
})
})
})
The key to accessing command return values is using .then() after the command.
It's annoying but necessary because Cypress ensures that the web page has received data from the server before evaluating the element text.
Since the test runs faster than web page fetches data, it can easily evaluate the text before the page is fully populated.
You have to convert your texts to numbers and then add it. You can simply add + in front of the number to convert them into Integers. Also I have added trim() in case your strings have any unwanted spaces.
And('I add up all the total on hand', () => {
const sumOnHand =
+itemListPage.totalOnHandAmsterdam().trim() + +itemListPage.totalOnHandDelft().trim()
cy.log(sumOnHand)
})
You could set the function results as aliases.
Since the code is asynchronous, access it within cy.then().
totalOnHandAmsterdam() {
cy.get(':nth-child(2) > .dx-grandtotal > span')
.invoke('text')
.then(parseInt)
.as('amsterdamTotal') // alias will set this.amsterdamTotal
}
totalOnHandDelft() {
return cy.get(':nth-child(11) > .dx-grandtotal > span')
.invoke('text')
.then(parseInt)
.as('defltTotal') // alias will set this.delftTotal
}
And("I add up all the total on hand", function() { // use function() to access "this"
po.totalOnHandAmsterdam()
po.totalOnHandDelft()
cy.then(() => {
const sumOnHand = this.amsterdamTotal + this.defltTotal;
cy.log(sumOnHand)
})
})

nodejs convert objects in json to lines of strings and store the lines in a text file

I need help converting an array of json into lines of text
This is the sample array
var links =
[
{
"file_name": "Night shot of Barcelona, Spain.jpg",
"url": "https://i.imgur.com/uMEags4.jpg",
"full_link": "https://www.reddit.com/r/pics/comments/f4ppj9/night_shot_of_barcelona_spain/",
"downloadId": "cln85w0k4zogv6a"
},
{
"file_name": "Nostalgia.jpg",
"url": "https://i.redd.it/lyxuxk2cemk41.jpg",
"full_link": "https://www.reddit.com/r/pics/comments/fdaaua/nostalgia/",
"downloadId": "cln85w0k4zogv6c"
}
]
I want this in a text format like this:
https://i.imgur.com/uMEags4.jpg
out=Night shot of Barcelona, Spain.jpg
https://i.redd.it/lyxuxk2cemk41.jpg
out=Nostalgia.jpg
There are about 10k objects in the array.I'm planning to download them using aria2c
I'm using a loop like this.
links.forEach((link) => {
})
but I don't know what the next steps are.
EDIT
I used #uday method. But I stored the string in a variable and then stored the string variable into a file
ultUrls.map((link, i) => {
txtstr += `${link.url}\n\tout=${link.file_name} \n\tdir=${dir}\n`
})
fs.writeFile(datapath + '\\'+ 'aria2clinks' + '\\' + textname, txtstr, function(err) {
if (err) {
console.log(err);
}
});
You can use map / forEach to loop through links and write to a file(s) using fs
const fs = require('fs')
links.map((link, i) => {
let data = `${link.url}\nout=${link.file_name} \n\n`
// if you want to write each json data in to new file // replace fileName_${i} as first arg in appendFile()
fs.appendFile('sample.txt', data, function (err) {
if (err) throw err;
});
})
output: sample.txt
https://i.redd.it/lyxuxk2cemk41.jpg
out=Nostalgia.jpg
https://i.imgur.com/uMEags4.jpg
out=Night shot of Barcelona, Spain.jpg

How do I remove this symbol from the CSV file that is generated by Puppeteer?

When I generate the CSV file each "item" output comes out with a  symbol. How would I go about removing this with my code. I tried to change it to utf-8 because I read that might be what's causing it. Any ideas? Example:
const products = await page.$$('.item-row');
Promise.all(products.map(async product => {
// Inside of each product find product SKU, it's human-readable name, and it's price
let productId = await product.$eval(".custom-body-copy", el => el.innerText.trim().replace(/,/g,' -').replace('Item ', ''));
let productName = await product.$eval(".body-copy-link", el => el.innerText.trim().replace(/,/g,' -'));
let productPrice = await product.$eval(".product_desc_txt div span", el => el.innerText.trim().replace(/,/g,' -'));
// Format them as a csv line
return productId + ',' + productName + ',' + productPrice + ',';
})).then(lines => {
// Write the lines to a file
fs.writeFileSync("products.csv", lines.join('\n'), 'utf-8');
browser.close();
});
});
There are probably better solutions, but the first thing that comes to mind to to change the string to an array with split() and then .map through and test for the à ascii code and change back to a string with join() like this:
const strToChange = 'My string with an à char';
const charWeDoNotWant = 'Ã'.charCodeAt();
const toFixArr = strToChange.split('');
fixedArr = toFixArr.map(char =>
char.charCodeAt() === charWeDoNotWant ? '' : char
);
const fixedStr = fixedArr.join('');
console.log(`String without Ã: ${fixedStr}`);

How do I get the input text (not just the value) from a Material for Web select component?

In the docs, it specifies how to get the index and data-value, but not the input text:
import {MDCSelect} from '#material/select';
const select = new MDCSelect(document.querySelector('.mdc-select'));
select.listen('MDCSelect:change', () => {
alert(`Selected option at index ${select.selectedIndex} with value "${select.value}"`);
});
The following assumes you have more than one MDCSelect to initiate
import {MDCSelect} from '#material/select';
const selectElements = [].slice.call(document.querySelectorAll('.mdc-select'));
selectElements.forEach((selectEl) => {
const select = new MDCSelect(selectEl);
select.listen('MDCSelect:change', (el) => {
const elText = el.target.querySelector(`[data-value="${select.value}"`).innerText;
console.log(`Selected option at index ${select.selectedIndex} with value "${select.value}" with a label of ${elText}`);
});
});

Renaming single file names while copying entire folder using copyTpl

My yeoman generator copies files from template to destination path:
this.fs.copyTpl(
this.templatePath(),
this.destinationPath(), {
appName: this.props.appName
});
During project generation, I need to assign value of this.props.appName to some of filenames.
Unfortunately I can't do this that way like I could do inside this files:
<%=appName%>-project.sln
All files that need to be renamed have appTemplate in their names, so what I need to do is simply replace appTemplate with value of this.props.appName.
Can I somehow configure copyTpl to rename some of files while copying them to another destination?
OK, I found a solution. According to yeoman docs:
Any generator author can register a transformStream to modify the file path and/or the content.
Using this method:
this.registerTransformStream();
What that means is I can pipe all generated files through some script:
var rename = require("gulp-rename");
//other dependecies...
module.exports = yeoman.Base.extend({
//some other things generator do...
writing: function() {
var THAT = this;
this.registerTransformStream(rename(function(path) {
path.basename = path.basename.replace(/(666replacethat666)/g, THAT.props.appName);
path.dirname = path.dirname.replace(/(666replacethat666)/g, THAT.props.appName);
}));
this.fs.copyTpl(
this.templatePath(),
this.destinationPath(), {
appName: this.props.appName
});
}
});
This script will pipe all files through gulp-rename, changing 666replacethat666 to something more intelligent.
If you cannot use registerTransformStream because you are using the composeWith() feature in Yeoman (which disconnects transform stream registrations), you can use the processDestinationPath, which works when you select multiple files (not when you specify a specific file in the first argument, for some reason).
this.fs.copyTpl(
this.templatePath("**/{.*,*}"),
this.destinationPath(),
{ /* usually your prompt answers are here */ },
{},
{
processDestinationPath: (filePath: string) =>
filePath.replace(/somedir\/a-file.js/g, 'newdir/better-filename.js'),
},
);
Source to documentation options: https://yeoman.github.io/generator/actions_fs.html#.copyTemplate
Which is based on https://github.com/SBoudrias/mem-fs-editor#copyfrom-to-options-context-templateoptions-
registerTransformStream with gulp-rename is still an issue. However, I get it working with glob.
const glob = require('glob');
writing() {
const files = glob.sync('**', { dot: true, nodir: true, cwd: this.templatePath() })
for (let i in files) {
this.fs.copyTpl(
this.templatePath(files[i]),
this.destinationPath( this.props.destinationFolderPath + '\\' + files[i].replace(/__fileName__/g,this.props.fileName)),
this.props
)
}
}
After copy, iterate over the paths of the output dir and regex replace all occurrences.
const getReplacement = (base, pathRel, match, replace) => {
let pathRelNew = pathRel.replace(match, replace);
let oldPathAbs = path.join(base, pathRel);
let newPathAbs = path.join(base, pathRelNew);
if (oldPathAbs != newPathAbs) {
return {
oldPath: oldPathAbs,
newPath: newPathAbs
}
}
}
const getReplacementsRecursive = (base, match, replace, replacements = []) => {
let pathsRel = fs.readdirSync(base);
pathsRel.forEach(pathRel => {
if (fs.statSync(path.join(base, pathRel)).isDirectory()) {
replacements = getReplacementsRecursive(path.join(base, pathRel), match, replace, replacements);
var replacement = getReplacement(base, pathRel, match, replace)
if (replacement) replacements.push(replacement);
} else {
var replacement = getReplacement(base, pathRel, match, replace)
if (replacement) replacements.push(replacement);
}
});
return replacements;
};
function replaceMatches(dir, match, replace) {
var replacements = getReplacementsRecursive(dir, match, replace);
replacements.forEach(function(replacement) {
fs.renameSync(replacement.oldPath, replacement.newPath);
});
}
module.exports = class extends Generator {
// ...
writing() {
// don't forget to set the output directory
let OUTPUT_DIR = "./out";
// this.fs.copyTpl(...);
// setTimeout is used to give some time for the copyTpl to finish
setTimeout(
() => {
var match = new RegExp( "666replacethat666", 'g' );
replaceMatches(OUTPUT_DIR, match, this.props.appName);
}, 1000);
}
}

Resources