Here is the code that tries to submit a multipart/form-data according to documentation:
var request = require('request');
var req = request.post('http://echo.httpkit.com',
function (err, resp, body) {
console.log(body);
});
var form = req.form()
form.append('name', 'value')
The response is:
411 Length Required
I would recommend using the form-data library: https://github.com/felixge/node-form-data
npm install form-data
Then set your code to something like so:
var FormData = require('form-data');
var request = require('request');
var form = new FormData();
form.append('name', 'value')
form.submit('http://echo.httpkit.com', function(e, r){
console.log(e,r)
});
var r = request.defaults('headers':{'Content-Length':contentlen} });
var post = r.post(url, function(err, response){});
var form = post.form();
form.append('param1', param1);
form.append('data', fs.createReadStream(pathtofile));
here contentlen is int
Related
fairly noobish at aes-ecb encryption, apologies in advance.
I am trying to make an API call to a server that requires a encrypted payload.
I'm having a little trouble in the sequence in which i encrypt my data converting between base64 and ascii.
When i run the below code, i keep getting
The header content contains invalid characters.
I suspect it may be the way i'm converting the encrypted data between types but not entirely sure. Any thoughts would be really appreciated.
var request = require("request");
var aesjs = require('aes-js');
var apiKey = "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX";
var privateKey = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
var method = 'GET';
var url = 'https://apiurl.com';
var agent = "app /1.0 Android/8.1.0 (Android)";
var options = {
method: method,
url: url,
headers:
{ 'X-API-KEY': apiKey,
'X-CSN-USER-AGENT': agent,
'apiData' : GetAndEncryptApiData(method, url)
}
};
request(options, function (error, response, body) {
if (error) throw new Error(error);
console.log(body);
});
function GetAndEncryptApiData(method, url){
var isoTimestamp = new Date().toISOString();;
var text = `uri:${url} \nmethod:${method}\ntimestamp:${isoTimestamp}`;
var key = Buffer.from(privateKey, 'base64');
var aesEcb = new aesjs.ModeOfOperation.ecb(key);
var textToBytes = aesjs.utils.utf8.toBytes(text);
var paddedData = aesjs.padding.pkcs7.pad(textToBytes);
var encryptedData = aesEcb.encrypt(paddedData);
var output = Buffer.from(encryptedData).toString('ascii');
return output;
}
Requirement:
I need to make an api call with the file uploaded by the user.My server is node and I use request module for making api calls.
Below is the code when user uploads a file and submits it.
if(queryData.sub == "upload"){
var input = {};
var formidable = require('formidable');
var form = new formidable.IncomingForm();
form.parse(request, function (err, fields, files) {
var fs = require('fs');
fs.readFile(files.filetoupload.path, function(err, data) {
input.x_file_content = data;
client.API.ATTACHMENTS.uploadFile(input).then(function(resp){
var str = settings.layoutParsing(resp);
response.write(str);
response.end();
})
});
});
}
}
In upload file function i use FormData to set the file and send it while making api call .Below is the code:
if (request.x_file_content) {
var FormData = require('form-data');
var formData = new FormData();
formData.append('file', request.x_file_content);//No I18N
req_body = formData;
}
...
var httpclient = require('request');
httpclient({
uri : baseUrl,
method : request.type,
headers : api_headers,
responseType : responseType,
body : req_body
},function(error,response,body){
Problem:
but the file was not successfully sent and multipart content required error is thrown by the api server.
Can anyone point out what mistake Im doing.
Thanks !
found out the mistake,
setting header and replaced
fs.readFile(files.filetoupload.path, function(err, data) {
with readStream = fs.createReadStream("path to file");
rectified code :
input.x_file_content = readStream;
..
var FormData = require('form-data');
form_Data = new FormData();
form_Data.append('file', request.x_file_content);//No I18N
req_body = form_Data;
api_headers = form_Data.getHeaders();
I'm make a webscraper and I already know how to scrap some data and convert them to Json with this code I made :
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
var url = 'http://www.footmercato.net/';
request(url, function(err, resp, body) {
if (!err) {
var $ = cheerio.load(body);
var data = [];
var i = 1;
$('.text').each(function(i, element) {
var article = $('p');
var jsObject = { title : "", article : "", date : "" };
var articleTxt = article.text();
jsObject.article = articleTxt;
data.push(jsObject);
})
var json = JSON.stringify(data);
fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err) {
console.log('File successfully written!');
})
}
});
app.listen('8080');
But I would like to navigate to the website I'm scraping, fill out form and going to others pages.
Does somebody know if i can do it with cheerio or how I can add it to my existing code ?
Thanks
You can use webdriverio actually he will open a browser window, and then you can manipulate the dom through the webdriverio api to handle forms mouse clicks, and navigate from one page to an other.
var webdriverio = require('webdriverio');
var options = {
desiredCapabilities: {
browserName: 'firefox'
}
};
webdriverio
.remote(options)
.init()
.url('http://www.google.com')
.getTitle().then(function(title) {
console.log('Title was: ' + title);
})
.end();
I hope to scrape urls from this website by using the code below:
var request = require("request");
cheerio = require("cheerio");
urls = [];
request("http://news.sabay.com.kh/topics/sport", function(err, resp, body){
if(!err && resp.statusCode ==200){
var $ = cheerio.load(body);
$(".article","h4.title").each(function(){
var url = this.attr("href");
urls.push(url);
});
console.log(urls);
}
});
but I cannot get the result. When I run I got this
$ node server.js
[]
First, use a proper CSS selector :
.article h4.title > a
Then, use the proper field :
var url = this.attribs.href
Which gives :
var request = require("request");
cheerio = require("cheerio");
urls = [];
request("http://news.sabay.com.kh/topics/sport", function(err, resp, body){
if(!err && resp.statusCode ==200){
var $ = cheerio.load(body);
$(".article h4.title > a").each(function(){
var url = this.attribs.href;
urls.push(url);
});
console.log(urls);
}
});
and outputs :
[ 'http://news.sabay.com.kh/article/546826',
'http://news.sabay.com.kh/article/546763',
'http://news.sabay.com.kh/article/546520',
'http://news.sabay.com.kh/article/546568',
'http://news.sabay.com.kh/article/546460',
'http://news.sabay.com.kh/article/546448',
'http://news.sabay.com.kh/article/545674',
'http://news.sabay.com.kh/article/546235',
'http://news.sabay.com.kh/article/545698',
'http://news.sabay.com.kh/article/546091' ]
do i need to keep each link in a array within "request({..})", and then display it or work on the outside of "request({..})", this would be my code but does not work, any idea?
var request = require("request");
var cheerio = require("cheerio");
var arrayLinks = [];
request({
uri: "http://www.some-url.com",
}, function(error, response, body) {
var $ = cheerio.load(body);
$("a").each(function() {
var link = $(this);
arrayLinks.push(link.attr("href"));
});
});
arrayLinks.forEach(function(link){console.log(link)});
For example:
var request = require("request");
var cheerio = require("cheerio");
var arrayLinks = [];
request({
uri: "http://www.some-url.com",
}, function(error, response, body) {
// Some logic.
linkTheArray()
});
function linkTheArray() {
arrayLinks.forEach(function(link){console.log(link)});
}
Now you can run it after the request is done. There is one other way, but it is pretty ugly. You can run a timeout function, until you get some data in the array