i'm doing a web scraping bot thanks to puppeteer, and i want to verify a field of a dataLayer on some url thanks to cloud functions on Google Cloud Plateform.
right now i'm using the await page.$('dataLayer') and it return a "true", so it find the dataLayer.
there is the dataLayer on my browser (not puppeteer)
my code:
exports.datalayer = async(page) => {
//verfify if there is a dataLayer
let dl = await page.$('dataLayer')
if(dl =! null){
console.log('DataLayer found')
//want to do something like that
for(let key in dl){
let value = dl[key]
console.log('key: ' + key + ' value: ' + value)
}
//or like that
dl.forEach(element => {
console.log(element)
});
}else{
console.log('error with the dataLayer')
}
}
In order to catch the OnetrustActiveGroups data.
when i do the forEach method i get this error:
and for the for method i get this error:
better use the page.evaluate('dataLayer')
then stringify it or use it as you want :)
and with the evaluate the for(let item in dl) works fine !
I am currently forwarding cloudwatch alarms to slack using a nodejs lambda function I found on github, however, when it receives a custom alert from one of our customers datacenters, the string is not in a JSON format, so it will not print the alert.
I am looking at rewriting the handleCatchAll() function to handle the string and just forward it to slack in its current format, but am having issues identifying the object.
Below is the code i am trying to edit, I need to write an if statement that will identify the object, say "if JSON.stringify fails, then render the message without JSON.stringify and input its value into 'description'"
var description = ""
if ()
{
else if ()
{
for(key in message) {
var renderedMessage = typeof message[key] === 'object'
? JSON.stringify(message[key])
: message[key]
description = description + "\n" + key + ": " + renderedMessage
}
}
}
You can use this script to convert text input into json if the text is a valid JSON object, or else keep the original text.
const isJSON = str => {
try {
return (JSON.parse(str) && !!str);
} catch (e) {
return false;
}
}
const regulizeRequestBody = body => isJSON(body) ? JSON.parse(body) : body;
I'm going to be honest. I'm way in over my head here.
I need to scrape data from a dynamic site for my employer. Before the data is visible on the page, there are some clicks and waits necessary. Simple PHP scraping won't do. So I found out about this NodeJS + PhantomJS combo. Quite a pain to set up, but I did manage to load a site, run some code and get a result.
I wrote a piece of jQuery which uses timeout loops to wait for some data to be loaded. Eventually I get a js object that I want to write to a file (JSON).
The issue I'm facing.
I build up the the js object inside the PhantomJS .evaluate scope, which runs in a headerless browser, so not directly in my Node.JS server scope. How do I send the variable I built up inside evaluate back to my server so I can write it to my file?
Some example code (I know it's ugly, but it's for illustrative purposes). I use node-phantom-simple as a bridge between Phantom and Node
var phantom = require('node-phantom-simple'),
fs = require('fs'),
webPage = 'https://www.imagemedia.com/printing/business-card-printing/'
phantom.create(function(err, ph) {
return ph.createPage(function(err, page) {
return page.open(webPage, function(err, status) {
page.onConsoleMessage = function(msg) {
console.log(msg);
};
console.log("opened site? ", status);
page.evaluate(function() {
setTimeout(function() {
$('.price-select-cnt').eq(0).find('select').val('1266').change()
timeOutLoop()
function timeOutLoop() {
console.log('looping')
setTimeout(function() {
if ($('#ajax_price_tool div').length != 6) {
timeOutLoop()
} else {
$('.price-select-cnt').eq(1).find('select').val('25')
$('.price-select-cnt').eq(2).find('select').val('Premium Card Stock')
$('.price-select-cnt').eq(3).find('select').val('Standard').change()
timeOutLoop2()
}
}, 100)
}
function timeOutLoop2() {
console.log('looping2')
setTimeout(function() {
if ($('.pricing-cost-cnt').text() == '$0' || $('.pricing-cost-cnt').text() == '') {
timeOutLoop2()
} else {
var price = $('.pricing-cost-cnt').text()
console.log(price)
}
}, 100)
}
}, 4000)
});
});
});
});
function writeJSON(plsWrite) {
var key = 'file'
fs.writeFile('./results/' + key + '.json', plsWrite, 'utf8', function() {
console.log('The JSON file is saved as');
console.log('results/' + key + '.json');
});
}
So do do I write the price this code takes from the website, get it out of the evaluate scope and write it to a file?
I was a PHP developer, I'm trying to write code using node.js for praticing. M am confuse about node.js when I execute my program, line of code jump to next line.
This is the part of my code:
//article.js
var article_model = require('../models/article_model');
var comment_model = require('../models/comment_model');
var list_article,list_comment;
app.set('views','./views/article/');
app.set('view engine','ejs');
app.get('/list_article',csrfProtection,function(req,res){
master_model.get_article(req,xparams,function(status,result,total_data){
list_article = result.data;
});
console.log(list_article);
master_model.get_comment(req,xparams,function(status,result,total_data){
list_comment = result.data;
});
console.log(list_comment);
var params = {
title : "Article List",
data_article : list_article,
};
res.render('content.ejs',params);
});
and then
//master_model.js
exports.get_article = function (req,hash, fn) {
var auths = {
user : api_server["auth_username"],
pass : api_server["auth_password"],
}
request.get({url:"http://myapi.com/article/latest", auth:auths } , function(err,httpResponse,body) {
if (!err && httpResponse.statusCode == 200) {
var temp = JSON.parse(body);
if (temp.status == 1){
result_data = {status:1, message : temp.message ,data : temp.data};
return fn(true,result_data,1);
}else if(temp.status == 0){
result_data ={ status:0, message : temp.message};
return fn(false,result_data,0);
}
}else{
result_data ={ status:0, message : "error, please try again"};
return fn(false,result_data,0); //something problem to API
}
})
};
exports.get_comment = function (req,hash, fn) {
var auths = {
user : api_server["auth_username"],
pass : api_server["auth_password"],
}
request.get({url:"http://myapi.com/comment/latest", auth:auths } , function(err,httpResponse,body) {
if (!err && httpResponse.statusCode == 200) {
var temp = JSON.parse(body);
if (temp.status == 1){
result_data = {status:1, message : temp.message ,data : temp.data};
return fn(true,result_data,1);
}else if(temp.status == 0){
result_data ={ status:0, message : temp.message};
return fn(false,result_data,0);
}
}else{
result_data ={ status:0, message : "error, please try again"};
return fn(false,result_data,0); //something problem to API
}
})
};
when i run my code, and open browser, output data is blank, when i refresh again my browser show ouput my data (artcile list, and commment list)
and i look my console if first run
output : {
id : 1
title : title 1..
...
..
}
undefined
If i refresh my browser again, all output complete to show (not show undefined)
My question:
How to make my code run the process step by step until the process end and deliver to views?
Any problem with my code?
Thanks!
To address your initial question properly, you can actually go step by step, using breakpoints. In the latest node 7 and now 6 you can use Chrome Dev Tools to debug a node application. You need to set some breakpoints and watch variables to help you understand:
execution order
the precise values of variables at key moments
I still think breaking the big program into smaller programmes would be helpful and reduce surprises, but learning to debug properly is also really useful!
You just run the program with node --inspect index.js, and then a link will be displayed on the console that you load up in Google Chrome Browser.
There is a more thorough guide, which might help.
The official documentation is also worth a read.
I am trying to test Geolocation in HTML 5 in my browser. But for some reason, it is not displaying the lat and long for my place. Nor even the error message
Why is the lat and long not displaying. please assist. I tried on Chrome and FF. same result. Does it not display result all time or takes time
window.onload = checkLocation;
function checkLocation() {
//we put the if condition that will check if the browser supports geplocationn feature or not
if(navigator.geolocation) {
//if yes, then call the getCurremtPosition method and pass in a handler function ()
navigator.geolocation.getCurrentPosition(showLocation, showError);
// And we’re calling the geolocation object’s getCurrentPosition method with one argument, the success callback.
}else {
alert(' Hey No locator for your browser');
}
}
function showLocation(position) {
//position parameter is passed so as to get getCurrentPosition’s handler is passed a position, which contains the latitude and longitude of your location
// get the lat and lon of location from the cords object
var lat = position.coords.lat;
var lon = position.coords.lon;
var content = document.getElementById('myLocation');
content.innerHTML = ' Your latitude position is: ' + lat + ' and your Longtitude position is: ' + lon + '</br>';
}
function showError(error) {
var errorList = {
//We create an object with three properties named zero to three.
//These properties are strings with an error message we want to associate with each code
0: 'Unknown error',
1: 'Permission denied by user',
2: 'Position not available',
3: 'Request timed out'
};
var errorMsg = errorList[error.code];// use error code property to access the error list []
if(error.code == 0 || error.code == 2) {
errorMsg = errorMsg + " " + error.message; // use erroo message property ro access the value in error list
}
var content = document.getElementById('myLocation');
content.innerHTML = erroMsg;
}
BTW i have found the issue. While calling the map api script, make sure the url starts with // and not http. Let google decide how it wants to serve the api file.