Node.js AUTHORIZE endpoints with Oauth2-Google passport - node.js

Question

I'm trying to develop a simple API in node and hiding it behind a google authentification layer, for that I'm using passport-google-oauth2 to enforce a google authentification to do some requests.
....
var GoogleStrategy = require('passport-google-oauth2').Strategy;
var passport = require('passport');
app.use(passport.initialize());
app.use(passport.session());
passport.serializeUser(function(user, done) {
done(null, user);
});
passport.deserializeUser(function(user, done) {
done(null, user);
});
passport.use(new GoogleStrategy({
clientID: CLIENT_ID,
clientSecret: CLIENT_SECRET,
callbackURL: "/oauth2callback",
passReqToCallback : true
},
function(request, accessToken, refreshToken, profile, done) {
console.log("passportUse access Profile ", accessToken);
done(null, accessToken);
}
));
app.get('/oauth2callback',
passport.authenticate( 'google', {
successRedirect: '/notesSuccess',
failureRedirect: '/notesNotSuccess'
}));
app.get('/notes',
passport.authenticate('google', { scope:
[ 'https://www.googleapis.com/auth/plus.login', 'https://www.googleapis.com/auth/drive'] }
), function(req, res) {
//res.json({notes: "This is your notebook. Edit this to start saving your notes!"})
})
app.get('/notesSuccess', function(req, res) {
res.json({notes: "This is your notebook. Edit this to start saving your notes!"})
})
app.get('/notesNotSuccess', function(req, res) {
res.json({notes: "AUTHENTICATION FAIL"})
})
app.get('/notes/API1', function(req, res) {
res.json({notes: "API1111111"})
})
....
The code is working more or less, at passport.use I do get all the google plus profile information and a token, so I assume that part is working.
The issue is that when I use the browser to visit localhost:8080/notes, I get the results as expected (/notesSuccess) when I login, if I'm in a private window I can see the google login page, working well so far.
My problem is if I do a query with postman using the token I was given or a new token with the same scopes I get only the login page (as if I had no Authentification header). If I query notes/API1 (no Oauth there) it works properly
Any ideas why the headers are not authentifing the query?