How to differentiate folders from files in storage.getFiles() response? - node.js

This is what I have in my bucket:
images/
image1.jpeg
I'm using the storage.bucket().getFiles() method:
const images = await storage.bucket().getFiles({ prefix: 'images' });
console.log(images);
And this is the response I'm getting:
You can see that I get 2 items.
1 for the images folder itself
And 1 for the actual image1.jpeg file
[
[
File {
_events: [Object: null prototype] {},
_eventsCount: 0,
_maxListeners: undefined,
metadata: [Object],
baseUrl: '/o',
parent: [Bucket],
id: 'images%2F',
createMethod: undefined,
methods: [Object],
interceptors: [],
projectId: undefined,
create: undefined,
bucket: [Bucket],
storage: [Storage],
kmsKeyName: undefined,
userProject: undefined,
name: 'images/',
acl: [Acl],
instanceRetryValue: true,
instancePreconditionOpts: undefined,
[Symbol(kCapture)]: false
},
File {
_events: [Object: null prototype] {},
_eventsCount: 0,
_maxListeners: undefined,
metadata: [Object],
baseUrl: '/o',
parent: [Bucket],
id: 'images%2Fimage1.jpeg',
createMethod: undefined,
methods: [Object],
interceptors: [],
projectId: undefined,
create: undefined,
bucket: [Bucket],
storage: [Storage],
kmsKeyName: undefined,
userProject: undefined,
name: 'images/image1.jpeg',
acl: [Acl],
instanceRetryValue: true,
instancePreconditionOpts: undefined,
[Symbol(kCapture)]: false
}
]
]
I would like to loop over the files in images[0] and be able to differentiate the folder from the files. How can I do it?

Tried this on live data in this document
with { prefix: 'images' } the entire bucket under the prefix is returned.
with { prefix: 'images/' } the entire folder under the prefix is returned.
with { prefix: 'images/image'} restrict the results to only the "files" in the given "folder"
If you want to list only files in the given folder try changing the prefix like this:
const images = await storage.bucket().getFiles({ prefix: 'images/image' });

Related

Puppeteer file upload with webkitdirectory

I have written a page with a form which has a file input field with webkitdirectory. Now I am trying to test it with Puppeteer. There appears to be a uploadFile('path/to/file') method which I was hoping would also accept directories. Unfortunately, it doesn't seem to do so.
This is the line in the test file:
await input.uploadFile('/path/to/directory')
When on the nodejs side I print the file (which is handled with formidable) I get this output:
File {
_events: [Object: null prototype] {},
_eventsCount: 0,
_maxListeners: undefined,
size: 0,
path:
'/tmp/uploads-dir--aohyI0/upload_9f55c1ec641fac3902faccf665cfac35',
name: '',
type: 'application/octet-stream',
hash: null,
lastModifiedDate: null,
_writeStream: null }
However when I manually upload the directory with chrome by hand I get this:
File {
_events: [Object: null prototype] {},
_eventsCount: 0,
_maxListeners: undefined,
size: 0,
path:
'/tmp/uploads-dir--dbCZZG/upload_ec20a6ffc079a3cba92d8114b7418527',
name:
'path/to/myfile.txt',
type: 'application/octet-stream',
hash: null,
lastModifiedDate: null,
_writeStream: null }
As you can see the name is blank on the first one and I do not receive any actual files.
Is there any way to test forms with webkitdirectory with Puppeteer?

How to connect to moto standalone server?

I just installed moto and tried to connect to the standalone server with the following node.js code:
const AWS = require('aws-sdk')
const ep = new AWS.Endpoint('http://127.0.0.1:5000')
const route53 = new AWS.Route53({endpoint: ep})
const params = {}
console.log(route53.listHostedZones(params))
The code sets AWS service's endpoint, which points to the standalone server of moto. And then sends listHostedZones request to the endpoint. The output of this program is
Request {
domain: null,
service:
Service {
config:
Config {
credentials: [Object],
credentialProvider: [Object],
region: undefined,
logger: null,
apiVersions: {},
apiVersion: null,
endpoint: [Object],
httpOptions: [Object],
maxRetries: undefined,
maxRedirects: 10,
paramValidation: true,
sslEnabled: true,
s3ForcePathStyle: false,
s3BucketEndpoint: false,
s3DisableBodySigning: true,
computeChecksums: true,
convertResponseTypes: true,
correctClockSkew: false,
customUserAgent: null,
dynamoDbCrc32: true,
systemClockOffset: 0,
signatureVersion: null,
signatureCache: true,
retryDelayOptions: {},
useAccelerateEndpoint: false },
endpoint:
Endpoint {
protocol: 'http:',
host: '127.0.0.1:5000',
port: 5000,
hostname: '127.0.0.1',
pathname: '/',
path: '/',
href: 'http://127.0.0.1:5000/',
constructor: [Object] },
_clientId: 1 },
operation: 'listHostedZones',
params: {},
httpRequest:
HttpRequest {
method: 'POST',
path: '/',
headers: { 'User-Agent': 'aws-sdk-nodejs/2.166.0 darwin/v6.2.2' },
body: '',
endpoint:
Endpoint {
protocol: 'http:',
host: '127.0.0.1:5000',
port: 5000,
hostname: '127.0.0.1',
pathname: '/',
path: '/',
href: 'http://127.0.0.1:5000/',
constructor: [Object] },
region: undefined,
_userAgent: 'aws-sdk-nodejs/2.166.0 darwin/v6.2.2' },
startTime: 2017-12-18T13:27:48.148Z,
response:
Response {
request: [Circular],
data: null,
error: null,
retryCount: 0,
redirectCount: 0,
httpResponse:
HttpResponse {
statusCode: undefined,
headers: {},
body: undefined,
streaming: false,
stream: null },
maxRetries: 3,
maxRedirects: 10 },
_asm:
AcceptorStateMachine {
currentState: 'validate',
states:
{ validate: [Object],
build: [Object],
afterBuild: [Object],
sign: [Object],
retry: [Object],
afterRetry: [Object],
send: [Object],
validateResponse: [Object],
extractError: [Object],
extractData: [Object],
restart: [Object],
success: [Object],
error: [Object],
complete: [Object] } },
_haltHandlersOnError: false,
_events:
{ validate:
[ [Object],
[Function: VALIDATE_REGION],
[Function: BUILD_IDEMPOTENCY_TOKENS],
[Function: VALIDATE_PARAMETERS] ],
afterBuild:
[ [Object],
[Function: SET_CONTENT_LENGTH],
[Function: SET_HTTP_HOST] ],
restart: [ [Function: RESTART] ],
sign: [ [Object] ],
validateResponse: [ [Function: VALIDATE_RESPONSE] ],
send: [ [Object] ],
httpHeaders: [ [Function: HTTP_HEADERS] ],
httpData: [ [Function: HTTP_DATA] ],
httpDone: [ [Function: HTTP_DONE] ],
retry:
[ [Function: FINALIZE_ERROR],
[Function: INVALIDATE_CREDENTIALS],
[Function: EXPIRED_SIGNATURE],
[Function: CLOCK_SKEWED],
[Function: REDIRECT],
[Function: RETRY_CHECK] ],
afterRetry: [ [Object] ],
build: [ [Function: buildRequest], [Function: sanitizeUrl] ],
extractData: [ [Function: extractData], [Function: extractRequestId] ],
extractError: [ [Function: extractError], [Function: extractRequestId] ],
httpError: [ [Function: ENOTFOUND_ERROR] ] },
emit: [Function: emit] }
As you can see in the above output, there's an ENOTFOUND_ERROR error in it. And I couldn't find any new connection in the console output of moto.
The way I start moto is moto_server route53:
> moto_server route53
* Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)
What's wrong with my configurations? I think there's something wrong with my node.js code. Do I misinterpret the meaning of endpoint?
const AWS = require('aws-sdk')
const ep = new AWS.Endpoint('http://127.0.0.1:5000')
const initParam = {
endpoint: ep,
region: 'ap-northeast-1'
}
const route53 = new AWS.Route53(initParam)
let params = {
CallerReference: 'unique_string_affh38h98hasd8f76a',
Name: 'www.example.com'
}
route53.createHostedZone(params, (err, data) => {
if (err) {
console.error(`createHostedZone err: ${err}`)
} else {
console.log(`createHostedZone data: ${JSON.stringify(data)}`)
}
})
I forgot to set region parameter. And the way I view the output of listHostedZones() was wrong. After correcting them, the code works correctly.

How to inspect network traffic and get the URL of resource requests?

I want to monitor the network of a page and get all the URLs of the JavaScript network events, similar to what PhantomJS' page.onResourceRequested is doing, but I couldn't figure it out how to do this with Google Chrome's Puppeteer.
I've been dabbling with Google Chrome's puppeteer, but I couldn't figure out how to make it work, as the output of it looks like this:
Page {
domain: null,
_events: {
request: [Function]
},
_eventsCount: 1,
_maxListeners: undefined,
_client: Session {
domain: null,
_events: {
'Page.frameAttached': [Function],
'Page.frameNavigated': [Function],
'Page.frameDetached': [Function],
'Runtime.executionContextCreated': [Function],
'Network.requestWillBeSent': [Function: bound _onRequestWillBeSent],
'Network.requestIntercepted': [Function: bound _onRequestIntercepted],
'Network.responseReceived': [Function: bound _onResponseReceived],
'Network.loadingFinished': [Function: bound _onLoadingFinished],
'Network.loadingFailed': [Function: bound _onLoadingFailed],
'Page.loadEventFired': [Function],
'Runtime.consoleAPICalled': [Function],
'Page.javascriptDialogOpening': [Function],
'Runtime.exceptionThrown': [Function],
'Security.certificateError': [Function],
'Inspector.targetCrashed': [Function]
},
_eventsCount: 15,
_maxListeners: undefined,
_lastId: 14,
_callbacks: Map {},
_connection: Connection {
domain: null,
_events: {},
_eventsCount: 0,
_maxListeners: undefined,
_url: 'ws://127.0.0.1:65343/devtools/browser/ca214df4-4357-4b8f-8552-a1524d6652ff',
_lastId: 17,
_callbacks: Map {},
_delay: 0,
_ws: [Object],
_sessions: [Object]
},
_targetId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923',
_sessionId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923:1'
},
_keyboard: Keyboard {
_client: Session {
domain: null,
_events: [Object],
_eventsCount: 15,
_maxListeners: undefined,
_lastId: 14,
_callbacks: Map {},
_connection: [Object],
_targetId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923',
_sessionId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923:1'
},
_modifiers: 0,
_pressedKeys: Set {}
},
_mouse: Mouse {
_client: Session {
domain: null,
_events: [Object],
_eventsCount: 15,
_maxListeners: undefined,
_lastId: 14,
_callbacks: Map {},
_connection: [Object],
_targetId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923',
_sessionId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923:1'
},
_keyboard: Keyboard {
_client: [Object],
_modifiers: 0,
_pressedKeys: Set {}
},
_x: 0,
_y: 0,
_button: 'none'
},
_frameManager: FrameManager {
domain: null,
_events: {
frameattached: [Function],
framedetached: [Function],
framenavigated: [Function]
},
_eventsCount: 3,
_maxListeners: undefined,
_client: Session {
domain: null,
_events: [Object],
_eventsCount: 15,
_maxListeners: undefined,
_lastId: 14,
_callbacks: Map {},
_connection: [Object],
_targetId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923',
_sessionId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923:1'
},
_mouse: Mouse {
_client: [Object],
_keyboard: [Object],
_x: 0,
_y: 0,
_button: 'none'
},
_frames: Map {
'232.1' => [Object]
},
_mainFrame: Frame {
_client: [Object],
_mouse: [Object],
_parentFrame: null,
_url: 'http://mytestdomain.com/',
_id: '232.1',
_defaultContextId: 4,
_waitTasks: Set {},
_childFrames: Set {},
_name: undefined,
_loadingFailed: false
}
},
_networkManager: NetworkManager {
domain: null,
_events: {
request: [Function],
response: [Function],
requestfailed: [Function],
requestfinished: [Function]
},
_eventsCount: 4,
_maxListeners: undefined,
_client: Session {
domain: null,
_events: [Object],
_eventsCount: 15,
_maxListeners: undefined,
_lastId: 14,
_callbacks: Map {},
_connection: [Object],
_targetId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923',
_sessionId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923:1'
},
_requestIdToRequest: Map {},
_interceptionIdToRequest: Map {
null => [Object], 'id-1' => [Object], 'id-2' => [Object], 'id-3' => [Object]
},
_extraHTTPHeaders: Map {},
_requestInterceptionEnabled: true,
_requestHashToRequestIds: Multimap {
_map: [Object]
},
_requestHashToInterceptions: Multimap {
_map: Map {}
}
},
_emulationManager: EmulationManager {
_client: Session {
domain: null,
_events: [Object],
_eventsCount: 15,
_maxListeners: undefined,
_lastId: 14,
_callbacks: Map {},
_connection: [Object],
_targetId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923',
_sessionId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923:1'
},
_emulatingMobile: false,
_injectedTouchScriptId: null
},
_tracing: Tracing {
_client: Session {
domain: null,
_events: [Object],
_eventsCount: 15,
_maxListeners: undefined,
_lastId: 14,
_callbacks: Map {},
_connection: [Object],
_targetId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923',
_sessionId: 'bbd35bf4-d3ce-4497-a2a0-8cc98b4f0923:1'
},
_recording: false,
_path: ''
},
_pageBindings: Map {},
_ignoreHTTPSErrors: false,
_screenshotTaskQueue: TaskQueue {
_chain: Promise {
undefined
}
},
_viewport: {
width: 800,
height: 600
}
}
Can you please tell me how can I get all the URLs of the JavaScript network events with Puppeteer?
Check out the sample that intercepts image requests. Easy to modify that to look at other types of resource requests:
await page.setRequestInterceptionEnabled(true);
page.on('request', request => {
if (/\.js$/i.test(request.url)) {
// request for js resource
}
request.continue();
});
await page.goto('https://example.com');
Came across this post and SetRequestInterceptionEnabled has been renamed to
page.setRequestInterception(value)
Here is a piece of code i found on the Documentation:
const puppeteer = require('puppeteer');
puppeteer.launch().then(async browser => {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', interceptedRequest => {
if (interceptedRequest.url.endsWith('.png') || interceptedRequest.url.endsWith('.jpg'))
interceptedRequest.abort();
else
interceptedRequest.continue();
});
await page.goto('https://example.com');
await browser.close();
});
NOTE Enabling request interception disables page caching.
Here is the URL for the puppeteer Documentation: Puppeteer Documentation
I think a solution more accurate to the question, and that does not interfere with the traffic, is using Page.on() listeners.
Something like:
page.on('request', (req) => console.log(req)); // 'requestFinished' and 'requestFailed' are other options
page.on('response', (res) => console.log(res));

Javascript Object properties (in Node) not getting logged or logged with a different name

I am using Node's formidable package from felixge. This is more a Javascript question than one specific to formidable, or so I think.
If I do a console.log on MYOBJ, I get the following:
{ file1:
File {
domain: null,
_events: {},
_eventsCount: 0,
_maxListeners: undefined,
size: 62464,
path: 'myDir/upload_e79d8d551721e2f399afbc39d5d5eaab.doc',
name: 'somefile.doc',
type: 'application/msword',
hash: null,
lastModifiedDate: Thu May 19 2016 20:22:24 GMT+0530 (IST),
_writeStream:
WriteStream {
_writableState: [Object],
writable: true,
domain: null,
_events: {},
_eventsCount: 0,
_maxListeners: undefined,
path: 'uploadDir/upload_1e0e9625e27f5c5172eaf5d18172f946.doc',
fd: null,
flags: 'w',
mode: 438,
start: undefined,
pos: undefined,
bytesWritten: 62464,
closed: true } } }
If I do a console.log, like so:
for (var filename in MYOBJ)
console.log(MYOBJ[filename]);
I get the following:
{ size: 62464,
path: 'myDir/upload_e79d8d551721e2f399afbc39d5d5eaab.doc',
name: 'somefile.doc',
type: 'application/msword',
mtime: '2016-05-19T14:52:24.129Z' }
My obvious question is:
Why are the other properties of "file1" not displayed? May be they are not the Object's "ownProperty"? Even so, why does "lastModifiedDate" become "mtime?"
Let me guess ... probably implementation for cosnole.log uses Object.getOwnPropertyDescriptor() and can list all non-enumerable values
var o = {}
Object.defineProperty(o, 'nonEnumerableValue', {value: 1})
console.log(o) // {}
console.log(o.nonEnumerableValue) // 1

Getting at all fields in the response to a monk findById call. nodejs/express/monk

Just cannot get my head around this after 10 hours of trying:
if I
users.findById(req.body.user_id,function(e,doc){});
and console.log the doc returned, all looks good:
{ _id: 54dcad6de4b01007caacb0cd,
username: 'realizertest',
password: '******************************',
first_name: 'Realizer',
second_name: 'Test',
display_name: 'Realizer Test',
email: 'system#realizerlabs.com' }
However, when trying to access the included fields, e.g. by:
user = users.findById(req.body.user_id,function(e,doc){});
var user_email = user.email;
I just get undefined. The user object looks like this:
{ col:
{ manager:
{ driver: [Object],
helper: [Object],
collections: [Object],
options: [Object],
_events: {} },
driver:
{ _construct_args: [],
_native: [Object],
_emitter: [Object],
_state: 2,
_connect_args: [Object] },
helper: { toObjectID: [Function], id: [Ob
name: 'users',
col:
{ _construct_args: [],
_native: [Object],
_emitter: [Object],
_state: 2,
_skin_db: [Object],
_collection_args: [Object],
id: [Object],
emitter: [Object] },
options: {} },
type: 'findOne',
opts: { fields: {}, safe: true },
domain: null,
_events:
{ error: [ [Function], [Function] ],
success: [ [Function], [Function] ] },
_maxListeners: 10,
emitted: {},
ended: false,
success: [Function],
error: [Function],
complete: [Function],
resolve: [Function],
fulfill: [Function],
reject: [Function],
query: { _id: 54dcad6de4b01007caacb0cd } }
I've also tried user.query.email but get the same result.
The findById obviously doesn't return a JSON object that I can use in this way.
How can I get at these fields?
It's an async call, so you need to use the callback, you can't assign that function to a variable:
users.findById(req.body.user_id,function(e,doc){
var user = doc;
console.log(user); //should see the object now, and access the props
});

Resources