I want to build a NodeJS based API that is backed with a pub-sub paradigm with e.g. Kafka. Here is a skeleton of what I want to do.
const express = require('express')
const serverApp = express()
serverApp.get('/book/:bookId', (req, res) => {
producer.send(JSON.stringify({
action: 'get',
message: req.params.bookId
}))
consumer.on('message', (data) => {
res.status(200).send(JSON.parse(data))
})
})
With the option above, the first invocation works but, the subsequent ones keep failing with ERR_HTTP_HEADERS_SENT.
Keeping the consumer.on outside of serverApp.get will need to have the req and res co-ordinated.
How do I implement such an API?
For example as skeleton
const express = require('express')
const serverApp = express()
const responses = Object.create(null);
consumer.on('message', (data) => {
const result = JSON.parse(data)
const res = responses[result.replyId]
res.status(200).send(result)
});
serverApp.get('/book/:bookId', (req, res) => {
const replyId = Math.random().toString(36).substr(2);
responses[replyId] = res;
producer.send(JSON.stringify({
action: 'get',
replyId,
message: req.params.bookId
}))
})
Related
Right now I have a front end react application using axios and and a backend server using node.js and express. I cannot for the life of me get my serp api data to post so that my front end can get it through axios and display the json data. I know how to get data to the front end but I am not a backend developer so this is proving to be incredibly difficult at the moment. I'm able to get the data from the the external api, I just don't know how to post it once I get it. Also I would not like to have all these request running on server.js so I created a controller but I think that is where it is messing up. Any help is appreciated
//pictures controller
const SerpApi = require('google-search-results-nodejs');
const {json} = require("express");
const search = new SerpApi.GoogleSearch("674d023b72e91fcdf3da14c730387dcbdb611f548e094bfeab2fff5bd86493fe");
const handlePictures = async (req, res) => {
const params = {
q: "Coffee",
location: "Austin, Texas, United States",
hl: "en",
gl: "us",
google_domain: "google.com"
};
const callback = function(data) {
console.log(data);
return res.send(data);
};
// Show result as JSON
search.json(params, callback);
//res.end();
}
// the above code works. how do i then post it to the server so that i can retrieve it to the backend?
module.exports = {handlePictures};
//server.js
const express = require('express');
const app = express();
const path = require('path');
const cors = require('cors');
const corsOptions = require('./config/corsOptions');
const { logger } = require('./middleware/logEvents');
const errorHandler = require('./middleware/errorHandler');
const cookieParser = require('cookie-parser');
const credentials = require('./middleware/credentials');
const PORT = process.env.PORT || 3500;
// custom middleware logger
app.use(logger);
// Handle options credentials check - before CORS!
// and fetch cookies credentials requirement
app.use(credentials);
// Cross Origin Resource Sharing
app.use(cors(corsOptions));
// built-in middleware to handle urlencoded form data
app.use(express.urlencoded({ extended: false }));
// built-in middleware for json
app.use(express.json());
//middleware for cookies
app.use(cookieParser());
//serve static files
app.use('/', express.static(path.join(__dirname, '/public')));
// routes
app.use('/', require('./routes/root'));
app.use('/pictures', require('./routes/api/pictures'));
app.all('*', (req, res) => {
res.status(404);
if (req.accepts('html')) {
res.sendFile(path.join(__dirname, 'views', '404.html'));
} else if (req.accepts('json')) {
res.json({ "error": "404 Not Found" });
} else {
res.type('txt').send("404 Not Found");
}
});
app.use(errorHandler);
app.listen(PORT, () => console.log(`Server running on port ${PORT}`));
//api/pictures.js
const picturesController= require('../../controllers/picturesController');
const express = require('express')
const router = express.Router();
// for POST request use app.post
router.route('/')
.post( async (req, res) => {
// use the controller to request external API
const response = await picturesController.handlePictures()
// send the response back to client
res.json(response)
})
module.exports = router;
You just need to return the result from SerpApi in your handlePictures function. To do this make a new Promise and when search.json runs callback do what you need with the results and pass it in resolve.
Your picturesController.js with an example of returning all results.
//pictures controller
const SerpApi = require("google-search-results-nodejs");
const { json } = require("express");
const search = new SerpApi.GoogleSearch(process.env.API_KEY); //your API key from serpapi.com
const handlePictures = async (req, res) => {
return new Promise((resolve) => {
const params = {
q: "Coffee",
location: "Austin, Texas, United States",
hl: "en",
gl: "us",
google_domain: "google.com",
};
const callback = function(data) {
resolve(data);
};
search.json(params, callback);
});
};
module.exports = { handlePictures };
Output:
And I advise you to change your API key to SerpApi to prevent it from being used by outsiders.
Since I don't have the full context of your App I can just assume the context. But given the fact that you already have wrapped the logic of calling the external API into a dedicated controller you can use it in the following way in an express app (used the hello world example from express):
// import your controller here
const express = require('express')
const app = express()
const port = 3000
// for POST request use app.post
app.get('/', async (req, res) => {
// use the controller to request external API
const response = await yourController.method()
// send the response back to client
res.json(response)
})
app.listen(port, () => {
console.log(`Example app listening on port ${port}`)
})
Here's an example how to execute the http request from the frontend:
const response = await fetch('http://localhost:3000') // result from res.json(response)
I am trying to do a GET request in order to retrieve some images from my Cloudinary account. But when I run the server, I get a 400 status code on my UI with reading
Cannot GET /
How can I solve this issue?
const express = require('express');
const dotenv = require('dotenv');
const cors = require('cors');
const { json } = require('body-parser');
const axios = require('axios');
const app = express();
app.use(cors());
app.use(json());
const { parsed: config } = dotenv.config();
const BASE_URL = `https://api.cloudinary.com/v1_1/${config.CLOUD_NAME}/resources/image`;
const auth = {
username: config.API_KEY,
password: config.API_SECRET,
};
app.get('/photos', async(req, res) => {
const response = await axios.get(BASE_URL + '/resources/image', {
auth,
params: {
next_cursor: req.query.next_cursor,
},
});
return res.send(response.data);
});
app.get('/search', async (req, res) => {
const response = await axios.get(BASE_URL + '/resources/search', {
auth,
params: {
expression: req.query.expression,
},
});
return res.send(response.data);
});
const PORT = 7000;
app.listen(PORT, console.log(`Server running on port ${PORT}`));
If you open your server URL in browser you will get Cannot GET / because you don't have base route.
It's not needed in most cases, since you don't access your node server from browser, it just run's in the background.
You generally display your frontend (Angular, React) in browser.
But if you don't like the message you can add a base route.
app.get('/', (req, res) => res.send('Hello World'));
I'm not sure what are you trying to achieve, but at least you won't get this error.
I'm following a tutorial at https://www.woolha.com/tutorials/node-js-google-cloud-pub-sub-basic-examples and having some difficulty..
I've the following code in server.js:-
const express = require('express');
const app = express();
const path = require('path');
const bodyParser = require('body-parser');
const dotenv = require('dotenv');
dotenv.config(); // Reads the .env file from the local folder.
// PubSub constant initialisation
const PubSub = require(`#google-cloud/pubsub`);
const pubsub = new PubSub();
const data = new Date().toString();
const dataBuffer = Buffer.from(data);
const topicName = 'sensehat-led-config';
app.use(bodyParser.urlencoded({ extended: true}));
// Tell the app to use the public folder.
app.use(express.static('public'));
app.get('/', (req,res) => {
res.send('Hello from App Engine!');
})
app.get('/submit', (req, res) => {
res.sendFile(path.join(__dirname, '/views/form.html'));
})
// Need to figure out how to get the css file to work in this. Can't be that hard.
app.get('/sensehat', (req, res) => {
res.sendFile(path.join(__dirname, '/views/sensehat.html'));
})
app.get('/sensehat-publish-message', (req, res) =>{
pubsub
.topic(topicName)
.publisher()
.publish(dataBuffer)
.then(messageId => {
console.log(`Message ${messageId} published`);
})
.catch(err => {
console.error('ERROR:', err);
});
})
app.post('/submit', (req, res) => {
console.log({
name: req.body.name,
message: req.body.message
});
res.send('Thanks for your message!');
})
// Listen to the App Engine-specified port, or 8080 otherwise
const PORT = process.env.PORT || 8080;
app.listen(PORT, () => {
console.log('Server listening on port ${PORT}...');
})
But when I run it I get a '500 Server Error', and looking at the Stackdriver logs I get the following error:-
TypeError: PubSub is not a constructor at Object.<anonymous>
I'm definitely a newbie at NodeJS and feeling my way around. After reading around I think the issue is coming from the
const PubSub = require(`#google-cloud/pubsub`);
const pubsub = new PubSub();
lines, but no idea how to rectify this.
You can try with latest versions of all libraries.
Dependencies in package.json
"dependencies": {
"#google-cloud/pubsub": "1.5.0",
"google-gax": "1.14.1",
"googleapis": "47.0.0"
}
Example code -
const {
PubSub
} = require('#google-cloud/pubsub');
const pubsub = new PubSub({
projectId: process.env.PROJECT_ID
});
module.exports = {
publishToTopic: function(topicName, data) {
return pubsub.topic(topicName).publish(Buffer.from(JSON.stringify(data)));
},
};
Calling file code
const PubSubPublish = require('path to your above file')
let publishResult = await PubSubPublish.publishToTopic(process.env.TOPIC_NAME, data)
Hope it helps!
You require the default export of #google-cloud/pubsub, but what look for is not in the default export.
Change the way you import PubSub to:
const {PubSub} = require(`#google-cloud/pubsub`);
Instead of:
const PubSub = require(`#google-cloud/pubsub`);
I have a small api I have built using Node.js and express.
I am trying to create a logger and I need log the request body AND response body.
app.use((req, res) => {
console.log(req);
res.on("finish", () => {
console.log(res);
});
});
"express": "^4.16.3",
However, i am not able to find the body in the req or res object. Please tell me how i can get them. thanks.
For res.body try the following snippet:
const endMiddleware = (req, res, next) => {
const defaultWrite = res.write;
const defaultEnd = res.end;
const chunks = [];
res.write = (...restArgs) => {
chunks.push(new Buffer(restArgs[0]));
defaultWrite.apply(res, restArgs);
};
res.end = (...restArgs) => {
if (restArgs[0]) {
chunks.push(new Buffer(restArgs[0]));
}
const body = Buffer.concat(chunks).toString('utf8');
console.log(body);
defaultEnd.apply(res, restArgs);
};
next();
};
app.use(endMiddleware)
// test
// HTTP GET /
res.status(200).send({ isAlive: true });
You need body-parser that will create body object for you in your request. To do that
npm install body-parser
var bodyParser = require('body-parser')//add this
app.use(bodyParser())//add this before any route or before using req.body
app.use((req, res) => {
console.log(req.body); // this is what you want
res.on("finish", () => {
console.log(res);
});
});
Ran into this problem but didn't like the solutions. An easy way is to simply wrap the original res.send or res.json with your logger.
Put this as middleware before your routes.
app.use(function responseLogger(req, res, next) {
const originalSendFunc = res.send.bind(res);
res.send = function(body) {
console.log(body); // do whatever here
return originalSendFunc(body);
};
next();
});
https://github.com/expressjs/express/blob/master/lib/response.js
res.send has signature of function(body) { return this; }
Here is a working example using the built in PassThrough stream. Remember to use the express.json() built in middleware to enable request body parsing.
After that, you need to intercept all writes to the response stream. Writes will happen on calling write or end, so replace those functions and capture the arguments in a separate stream.
Use res.on('finish', ...) to gather all the written data into a Buffer using Buffer.concat and print it.
const express = require('express');
const { PassThrough } = require('stream')
const app = express();
app.use(express.json());
app.use((req, res, next) => {
const defaultWrite = res.write.bind(res);
const defaultEnd = res.end.bind(res);
const ps = new PassThrough();
const chunks = [];
ps.on('data', data => chunks.push(data));
res.write = (...args) => {
ps.write(...args);
defaultWrite(...args);
}
res.end = (...args) => {
ps.end(...args);
defaultEnd(...args);
}
res.on('finish', () => {
console.log("req.body", req.body);
console.log("res.body", Buffer.concat(chunks).toString());
})
next();
})
app.use('/', (req, res) => {
res.send("Hello");
});
app.listen(3000);
install npm install body-parser
and use this snippet,
var express = require('express')
var bodyParser = require('body-parser')
var app = express()
// create application/json parser
var jsonParser = bodyParser.json()
to get json response
app.use(jsonParser, function (req, res) {
console.log(req.body); // or console.log(res.body);
})
There is ready made module https://www.npmjs.com/package/morgan-body
const express = require('express')
const morganBody = require("morgan-body")
const bodyParser = require("body-parser")
const app = express()
const port = 8888
// must parse body before morganBody as body will be logged
app.use(bodyParser.json());
// hook morganBody to express app
morganBody(app, {logAllReqHeader:true, maxBodyLength:5000});
app.get('/', (req, res) => {
res.send('Hello World!')
})
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`)
})
Hi was looking for same as complete log of request and response as middleware in express js. Found the solution as well w
/*Added by vikram parihar for log */
const moment = require('moment');
const rfs = require("rotating-file-stream");
const geoip = require('geoip-lite');
const { PassThrough } = require('stream')
let path = require('path');
const accessLogStream = rfs.createStream('access.log', {
interval: '1M', // rotate daily
compress: true,
path: path.join(__dirname, '../../log')
});
module.exports = function (req, res, next) {
try {
let geo = geoip.lookup(req.ip);
let country = geo ? geo.country : "Unknown";
let region = geo ? geo.region : "Unknown";
let log = {
"time": moment().format('YYYY/MM/DD HH:mm:ss'),
"host": req.hostname,
"ip": req.ip,
"originalUrl": req.originalUrl,
"geo": {
"browser": req.headers["user-agent"],
"Language": req.headers["accept-language"],
"Country": country,
"Region": region,
},
"method": req.method,
"path": req.path,
"url": req.url,
"body": req.body,
"params": req.params,
"query": req.query,
"response": {
"body": res.body
}
};
const defaultWrite = res.write.bind(res);
const defaultEnd = res.end.bind(res);
const ps = new PassThrough();
const chunks = [];
ps.on('data', data => chunks.push(data));
res.write = (...args) => {
ps.write(...args);
defaultWrite(...args);
}
res.end = (...args) => {
ps.end(...args);
defaultEnd(...args);
}
res.on('finish', () => {
log.response.body = Buffer.concat(chunks).toString()
accessLogStream.write(JSON.stringify(log) + "\n");
})
} catch (error) {
console.log(error)
next(error)
}
next();
}
I have an API that is trying to make an HTTP request to an API that streams and image back to the me, then either stream that image back to the client making the request to me or wait until the image has been streamed to me and send it all at once.
I am using Express and request-promise.
Here's a shortened version of my code.
const express = require('express');
const router = express.Router();
const request = require('request-promise');
const imgFunc = async () => {
try {
const response = await request.get({
method: 'GET',
uri: `http://localhost:8080`,
});
return response;
} catch(err) {
console.log(err);
}
};
router.get('/', async function(req, res, next) {
try {
const response = await imgFunc();
return res.send(response);
} catch (err) {
console.log(err);
}
});
module.exports = router;
The image that I get back is just what I assume is the binary data and I don't know if I need to do something at the request-promise level to make that right or when I send it back to the client.
The server that I have running at localhost:8080 mimics the actual server that I will be hitting when this is all said and done.
You could pipe the streams directly rather than using request-promise.
const express = require('express');
const router = express.Router();
const https = require('https');
router.get('/', function(req, res) {
const url = 'https://www.gravatar.com/avatar/2ea70f0c2a432ffbb9e5875039645b39?s=32&d=identicon&r=PG&f=1';
const request = https.get(url, function(response) {
const contentType = response.headers['content-type'];
console.log(contentType);
res.setHeader('Content-Type', contentType);
response.pipe(res);
});
request.on('error', function(e){
console.error(e);
});
});
module.exports = router;
Or using the request library on which request-promise is based:
const express = require('express');
const router = express.Router();
const request = require('request');
router.get('/', function(req, res) {
const url = 'https://www.gravatar.com/avatar/2ea70f0c2a432ffbb9e5875039645b39?s=32&d=identicon&r=PG&f=1';
request.get(url).pipe(res);
});
module.exports = router;
you could pipe the streams directly through axios
const express = require('express');
const router = express.Router();
const axios = require("axios");
router.get('/', function(req, res) {
const link = 'https://app.idesk360.com/media/logos/iDeskLoginTxt.png';
const arrayBuffer = await axios.get(link, {responseType: 'stream'});
const contentType = arrayBuffer.headers["content-type"];
res.setHeader('content-type', contentType);
arrayBuffer.data.pipe(res);
});
module.exports = router;