NodeJS Amazon AWS SDK S3 client stops working intermittently - node.js

I have NodeJS express web server that serves files from AWS S3. Most of the time this exact code works correctly and serves files for a wide verity of applications with large numbers of requests in Production. The NodeJS web server is running across multiple nodes on a docker swarm server.
After about 2-3 weeks this stops working. There is no response from S3Client GetObjectCommand, there no error returned or anything. This starts working again only after restarting the NodeJS Docker container.
I read the S3 SDK docs that indicate a that the SDK will retry automatically.
Each AWS SDK implements automatic retry logic.
Questions:
How can we make this code more resilient and not need a restart?
Is the error handling correct? I'm wondering why there is no seemingly no response or error returned at all in this situation.
Is it necessary to configure the re-try settings?
NodeJS version: node:lts-alpine
Module: #aws-sdk/client-s3
Controllers
AWS Controller
const consoleLogger = require('../logger/logger.js').console;
const { S3Client, GetObjectCommand } = require('#aws-sdk/client-s3');
const config = {
"credentials": {
"accessKeyId": "example",
"secretAccessKey": "example"
},
"endpoint": "example",
"sslEnabled": true,
"forcePathStyle": true
}
const s3client = new S3Client(config);
const awsCtrl = {};
awsCtrl.getObject = async (key) => {
// Get object from Amazon S3 bucket
let data;
try {
// Data is returned as a ReadableStream
data = await s3client.send(new GetObjectCommand({ Bucket: "example", Key: key }));
console.log("Success", data);
} catch (e) {
consoleLogger.error("AWS S3 error: ", e);
const awsS3Error = {
name: e.name || null,
status: e.$metadata.httpStatusCode || 500
};
throw awsS3Error;
}
return data;
}
module.exports = awsCtrl;
Files Controller
const queryString = require('query-string');
const consoleLogger = require('../logger/logger.js').console;
const httpCtrl = require('./http.ctrl');
const jwtCtrl = require('./jwt.ctrl');
const awsCtrl = require('./aws.ctrl');
filesCtrl.deliverFile = async (req, res) => {
/* Get object from AWS S3 */
let fileObjectStream;
try {
fileObjectStream = await awsCtrl.getObject(filePath);
} catch (e) {
consoleLogger.error(`Unable to get object from AWS S3`, e);
if (e.status && e.status === 404) {
result.error = `Not found`;
result.status = 404;
return res.status(result.status).json(result);
}
return res.status(e.status || 500).json(result);
}
const filename = lookupResponse.data.filename;
// Set response header: Content-Disposition
res.attachment(filename);
// API response object stream download to client
return fileObjectStream.Body.pipe(res);
}
API
const express = require('express');
const router = express.Router();
const filesCtrl = require('../../controllers/files.ctrl');
const filesValidation = require('../validation/files');
router.get('/:fileId', [filesValidation.getFile], (req, res, next) => {
return filesCtrl.deliverFile(req, res);
});

Related

Delivering image from S3 to React client via Context API and Express server

I'm trying to download a photo from an AWS S3 bucket via an express server to serve to a react app but I'm not having much luck. Here are my (unsuccessful) attempts so far.
The Workflow is as follows:
Client requests photo after retrieving key from database via Context API
Request sent to express server route (important so as to hide the true location from the client)
Express server route requests blob file from AWS S3 bucket
Express server parses image to base64 and serves to client
Client updates state with new image
React Client
const [profilePic, setProfilePic] = useState('');
useEffect(() => {
await actions.getMediaSource(tempPhoto.key)
.then(resp => {
console.log('server resp: ', resp.data.data.newTest) // returns ����\u0000�\u0000\b\u0006\
const url = window.URL || window.webkitURL;
const blobUrl = url.createObjectURL(resp.data.data.newTest);
console.log("blob ", blobUrl);
setProfilePic({ ...profilePic, image : resp.data.data.newTest });
})
.catch(err => errors.push(err));
}
Context API - just axios wrapped into its own library
getMediaContents = async ( key ) => {
return await this.API.call(`http://localhost:5000/${MEDIA}/mediaitem/${key}`, "GET", null, true, this.state.accessToken, null);
}
Express server route
router.get("/mediaitem/:key", async (req, res, next) => {
try{
const { key } = req.params;
// Attempt 1 was to try with s3.getObject(downloadParams).createReadStream();
const readStream = getFileStream(key);
readStream.pipe(res);
// Attempt 2 - attempt to convert response to base 64 encoding
var data = await getFileStream(key);
var test = data.Body.toString("utf-8");
var container = '';
if ( data.Body ) {
container = data.Body.toString("utf-8");
} else {
container = undefined;
}
var buffer = (new Buffer.from(container));
var test = buffer.toString("base64");
require('fs').writeFileSync('../uploads', test); // it never wrote to this directory
console.log('conversion: ', test); // prints: 77+977+977+977+9AO+/vQAIBgYH - this doesn't look like base64 to me.
delete buffer;
res.status(201).json({ newTest: test });
} catch (err){
next(ApiError.internal(`Unexpected error > mediaData/:id GET -> Error: ${err.message}`));
return;
}
});
AWS S3 Library - I made my own library for using the s3 bucket as I'll need to use more functionality later.
const getFileStream = async (fileKey) => {
const downloadParams = {
Key: fileKey,
Bucket: bucketName
}
// This was attempt 1's return without async in the parameter
return s3.getObject(downloadParams).createReadStream();
// Attempt 2's intention was just to wait for the promise to be fulfilled.
return await s3.getObject(downloadParams).promise();
}
exports.getFileStream = getFileStream;
If you've gotten this far you may have realised that I've tried a couple of things from different sources and documentation but I'm not getting any further. I would really appreciate some pointers and advice on what I'm doing wrong and what I could improve on.
If any further information is needed then just let me know.
Thanks in advance for your time!
Maybe it be useful for you, that's how i get image from S3, and process image on server
Create temporary directory
createTmpDir(): Promise<string> {
return mkdtemp(path.join(os.tmpdir(), 'tmp-'));
}
Gets the file
readStream(path: string) {
return this.s3
.getObject({
Bucket: this.awsConfig.bucketName,
Key: path,
})
.createReadStream();
}
How i process file
async MainMethod(fileName){
const dir = await this.createTmpDir();
const serverPath = path.join(
dir,
fileName
);
await pipeline(
this.readStream(attachent.key),
fs.createWriteStream(serverPath + '.jpg')
);
const createFile= await sharp(serverPath + '.jpg')
.jpeg()
.resize({
width: 640,
fit: sharp.fit.inside,
})
.toFile(serverPath + '.jpeg');
const imageBuffer = fs.readFileSync(serverPath + '.jpeg');
//my manipulations
fs.rmSync(dir, { recursive: true, force: true }); //delete temporary folder
}

node js not behaving the same way on local and Google App Egine

I'm developing an app to upload .las file to cesium ion.
I have modified this code https://github.com/CesiumGS/cesium-ion-rest-api-examples/blob/main/tutorials/rest-api/index.js
To pass a file from the browser.
It's working flawlessly when I run npm start on my local env.
When I try the same thing on the app Engine, I do not get the message about where the process is at. It's does upload the file though. It's just I can't monitor what is going on.
To explain what is going on below, I send a file from the client, then it's catch by app.post("/upload"
Then, it create asset on Ion, and then upload to S3, then it tell ion it's finished, then it monitor the tiling on Ion.
Then I call every second app.post("/progress" That is sending back the stat of things to the client.
I think there is probably a logic I'm missing, something basic I make totally wrong. I'm using one single service for both the backend and the frontend. Can this be a part of the problem ?
const express = require('express');
const app = express();
const port = process.env.PORT || 3001;
const fileUpload = require("express-fileupload");
const cors = require('cors');
var path = require('path');
const AWS = require('aws-sdk');
const fs = require('fs');
const rawdatafr = require('./lang/fr.json');
const rawdataen = require('./lang/en.json');
const axios = require('axios').default;
const accessToken = process.env.REACT_APP_ION_TOKEN;
const environment = process.env.NODE_ENV || 'production';
var urlLang = (environment === 'development') ? 'client/src/lang/' : 'lang/';
console.log('urlLang ? '+urlLang);
app.use(cors());
app.use(fileUpload({
useTempFiles: true,
safeFileNames: false,
preserveExtension: true,
tempFileDir: 'temp/'
}));
'use strict';
var messageFromLoc = rawdataen;
var input = null;
var filename = null;
var srcType = 'POINT_CLOUD';
var message = null;
var needMonitoring = false;
var assetMetadata = null;
var finished = null;
function resetGlobalvar(){
message = null;
needMonitoring = false;
assetMetadata = null;
finished = null;
input = null;
filename = null;
srcType = 'POINT_CLOUD';
}
async function creat_asset(){
finished = false;
message = 'create asset';
axios.post('https://api.cesium.com/v1/assets', {
name: filename,
description: '',
type: '3DTILES',
options: {
position:[ 2.29, 48.85, 0.1],
sourceType: srcType,
}
},{
headers: { Authorization: `Bearer ${accessToken}` }
})
.then(function (response) {
message = 'created successfully :> send to s3';
sendtos3(response.data);
})
.catch(function (error) {
console.log(error);
message = error;
});
}
async function sendtos3(response){
console.log('Asset created.');
message = 'send to s3';
try{
const uploadLocation = response.uploadLocation;
const s3 = new AWS.S3({
apiVersion: '2006-03-01',
region: 'us-east-1',
signatureVersion: 'v4',
endpoint: uploadLocation.endpoint,
credentials: new AWS.Credentials(
uploadLocation.accessKey,
uploadLocation.secretAccessKey,
uploadLocation.sessionToken)
});
let params = {
Body: fs.createReadStream(input),
Bucket: uploadLocation.bucket,
Key: uploadLocation.prefix+filename
};
let s3Response = await s3.upload(params).on('httpUploadProgress', function (progress) {
message = `${messageFromLoc.upload}: ${((progress.loaded / progress.total) * 100).toFixed(2)}%`;
console.log(`Upload: ${((progress.loaded / progress.total) * 100).toFixed(2)}%`);
}).promise();
// request successed
console.log(`File uploaded to S3 at ${s3Response.Bucket} bucket. File location: ${s3Response.Location}`);
message = `File uploaded to S3 at ${s3Response.Bucket} bucket. File location: ${s3Response.Location}`;
step3(response);
// return s3Response.Location;
}
// request failed
catch (ex) {
console.error(ex);
message = ex;
}
}
async function step3(response){
const onComplete = response.onComplete;
assetMetadata = response.assetMetadata;
message = 'step3';
axios.post(onComplete.url, onComplete.fields,{
headers: { Authorization: `Bearer ${accessToken}` }
})
.then(function (response) {
message = 'step3 done';
monitorTiling(assetMetadata);
})
.catch(function (error) {
console.log(error);
message = error;
});
}
async function monitorTiling(assetMetadata){
// console.log(response);
const assetId = assetMetadata.id;
message = 'monitorTiling';
axios.get(`https://api.cesium.com/v1/assets/${assetId}`,{headers: { Authorization: `Bearer ${accessToken}` }})
.then(function (response) {
// handle success
console.log('monitorTiling - success');
var status = response.data.status;
message = 'Tiling - success';
if (status === 'COMPLETE') {
console.log('Asset tiled successfully');
console.log(`View in ion: https://cesium.com/ion/assets/${assetMetadata.id}`);
message = 'Asset tiled successfully';
needMonitoring = false;
finished = true;
} else if (status === 'DATA_ERROR') {
console.log('ion detected a problem with the uploaded data.');
message = 'ion detected a problem with the uploaded data.';
needMonitoring = false;
finished = true;
} else if (status === 'ERROR') {
console.log('An unknown tiling error occurred, please contact support#cesium.com.');
message = 'An unknown tiling error occurred, please contact support#cesium.com.';
needMonitoring = false;
finished = true;
} else {
needMonitoring = true;
if (status === 'NOT_STARTED') {
console.log('Tiling pipeline initializing.');
message = 'Tiling pipeline initializing.';
} else { // IN_PROGRESS
console.log(`Asset is ${assetMetadata.percentComplete}% complete.`);
message = `Asset is ${assetMetadata.percentComplete}% complete.`;
}
}
})
.catch(function (error) {
// handle error
console.log(error);
message =error;
})
}
/*------- LISTEN FOR CALL TO UPLOAD AND START THE UPLOAD PROCESS ----------*/
app.post("/upload", (req, res) => {
if (!req.files) {
res.send("File was not found");
message = 'File was not found';
return;
}
input = req.files.file.tempFilePath;
filename = req.files.file.name;
emptyTempFolder('temp', input.replace('temp/', ''));
var ext = path.extname(filename);
if(ext=='.zip'){
srcType = 'CITYGML';
}
/*------- START UPLOAD PROCESS ----------*/
creat_asset();
});
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`)
})
/*------- LISTEN FOR PROGRESS TO UPLOAD ASSET ----------*/
app.get("/progress", (req, res) => {
// lang = req.get('Accept-Language').substring(0, 2).toLowerCase();
// if(lang=='fr'){
// messageFromLoc = rawdatafr;
// }
console.log('message ='+message);
if(needMonitoring){
monitorTiling(assetMetadata);
}
res.json({ message: message, done: finished, myAssetMetadata: assetMetadata });
if(finished){
resetGlobalvar();
}
});
/*--------------STATIC ----------------*/
app.use(express.static( path.join(__dirname, 'build' )));
And my app.yaml is like this :
runtime: nodejs14
env: standard
includes:
- env_variables.yaml
instance_class: B1
service: my-app
basic_scaling:
max_instances: 25
idle_timeout: 60m
I think this is from your instance(s). You're using basic scaling with up to 25 instances.
It looks like a combination of the following is happening
a) When you send a request to /progress, a new instance of your App is created which means all of the global variables are starting from their default values (the initial value of message is null).
b) Other times, a request to /progress is handled by an existing instance which was already processing an upload request and that request has completed and so the message says completed
You don't have this problem on your local environment because only 1 instance runs.
To test this theory, modify your app.yaml and set max_instances: 1. This is supposed to force the App to only use 1 instance which means subsequent requests should use an existing instance (which has the updated state of your global variables)

AWS Transcribe client does not provide an export named 'transcribeClient'

I'm trying to integrate AWS Transcribe in my Node.JS application. AWS S3 and Polly works fine, but AWS Transcribe does not. I'm using the example code of AWS.
When I want to start a transcribe job by the AWS example code I receive the following error: The requested module './libs/transcribeClient.js' does not provide an export named 'transcribeClient'
That was also the only file where I received the error that required is not defined. I wonder why it only happens with AWS transcribe but not with the other services as well? I'm also able to start a transcribe job via the AWS CLI.
That AWS Transcribe code does not work - transcribeClient.js:
const AWS_BUCKET_NAME="X"
const AWS_REGION="eu-central-1"
const AWS_ACCESS_KEY="XXX"
const AWS_SECRET_KEY="XXX"
// snippet-start:[transcribe.JavaScript.createclientv3]
const { TranscribeClient } = require('#aws-sdk/client-transcribe');
// Create anAmazon EC2 service client object.
const transcribeClient = new TranscribeClient({ AWS_REGION, AWS_ACCESS_KEY, AWS_SECRET_KEY });
module.exports = { transcribeClient };
That AWS Polly code works - pollyClient.js:
const AWS_BUCKET_NAME="X"
const AWS_REGION="eu-central-1"
const AWS_ACCESS_KEY="XXX"
const AWS_SECRET_KEY="XXX"
// snippet-start:[polly.JavaScript.createclientv3]
const { PollyClient } =require( "#aws-sdk/client-polly");
// Create an Amazon S3 service client object.
const pollyClient = new PollyClient({ AWS_REGION, AWS_ACCESS_KEY, AWS_SECRET_KEY});
module.exports = { pollyClient };
I'm looking forward to reading from you! Thanks!
I solved it. Now it's working with my Node.js 12 environment.
package.json
I changed "type": "modules" to "type": "commonjs".
transcribeClient.js needs to look like this:
Here I changed export to module.exports.
const { TranscribeClient } = require("#aws-sdk/client-transcribe");
const transcribeClient = new TranscribeClient({ AWS_REGION, AWS_ACCESS_KEY, AWS_SECRET_KEY});
module.exports = { transcribeClient };
transcribe_create_job.js needs to look like this:
Here I changed the import statement to require.
const { StartTranscriptionJobCommand } = require("#aws-sdk/client-transcribe");
const { transcribeClient } = require("./libs/transcribeClient.js")
// Set the parameters
const params = {
TranscriptionJobName: "test123",
LanguageCode: "en-GB", // For example, 'en-US'
MediaFormat: "webm", // For example, 'wav'
Media: {
MediaFileUri: "https://x.s3.eu-central-1.amazonaws.com/dlpasiddi.webm",
},
};
const run = async () => {
try {
const data = await transcribeClient.send(
new StartTranscriptionJobCommand(params)
);
console.log("Success - put", data);
return data; // For unit tests.
} catch (err) {
console.log("Error", err);
}
};
run();

Is uploading a file to S3 in NodeJS blocking?

I've been seeing performance issues on our application and I'm a bit unsure if uploading a file to S3 could block NodeJS.
I'm using express, formidable and aws-sdk.
Here's a middleware using formidable. This stores the file in req.file and continues to the next middleware that performs the upload to S3.
var formidable = require("formidable");
module.exports = function() {
return function(req, res, next) {
var form = new formidable.IncomingForm({
"keepExtensions": true,
"uploadDir": config.tempDir
});
form.parse(req, function(error, fields, files) {
if (error) {
return res.sendError("Error while parsing multipart-form " + error, 500);
}
req.files = files;
req.fields = fields;
next();
});
};
};
Here's the middleware that actually makes the request to S3 using AWS SDK
const AWS = require("aws-sdk");
const fs = require("fs");
const s3 = new AWS.S3(s3config.s3options);
const logger = require("logger");
function uploadFile(req, res, next) {
const requestId = getRequestIdFromRequestHeaders(req);
const file = options.file(req);
const contentType = options.contentType && options.contentType(req) || file.type;
const destinationPath = options.destinationPath(req);
req.s3uploaded = false;
logger.debug(requestId, "invoking uploadFile", contentType, destinationPath);
req.s3FilePath = "https://" + s3config.bucket + ".s3.amazonaws.com/" + destinationPath;
if (!options.writeConcern) {
logger.debug(requestId, "write concern not expected. Calling next");
next();
}
const stream = fs.createReadStream(file.path);
s3.upload({
"Bucket": s3config.bucket,
"ContentLength": file.size,
"Key": destinationPath,
"Body": stream,
"ContentType": contentType
}, s3config.s3options.uploadOptions, function(error) {
fs.unlink(file.path, error => {
if (error) {
logger.error(requestId, "Unable to remove file", file.path);
}
});
if (error) {
return next(error);
}
if (options.writeConcern) {
if (!req.s3uploaded) {
req.s3uploaded = true;
next();
}
}
}).on("httpUploadProgress", progress => {
logger.debug(requestId, "progress", progress.loaded, "of", progress.total);
if (progress.total !== undefined && progress.loaded === progress.total) {
logger.debug(requestId, "upload done, invoking next from httpUploadProgress");
if (!req.s3uploaded) {
req.s3uploaded = true;
next();
}
}
});
};
The documentation for the AWS SDK for JavaScript (v2) includes this statement on their Calling Services Asynchronously page (emphasis mine):
All requests made through the SDK are asynchronous. This is important to keep in mind when writing browser scripts. JavaScript running in a web browser typically has just a single execution thread. After making an asynchronous call to an AWS service, the browser script continues running and in the process can try to execute code that depends on that asynchronous result before it returns.

Nodejs Elastic benastalk refused to connect to upsteam/ upsteam prematurely closed

I am getting the following errors when running my application in elastic beanstalk: [error] 3636#0: *295 upstream prematurely closed connection while reading response header from upstream and [error] 3636#0: *295 connect() failed (111: Connection refused) while connecting to upstream Its strange because if I hit those routes independently it works fine. It only appears to error when firing those routes from my vuex action.
The following is the log from the AWS elastic beanstalk.
The following is the network tab when it hits my FFmpeg route:
The following is the generate video action as fired from vuex.
async [GENERATE_VIDEO]({state, rootState, dispatch, commit}){
const username = rootState.user.currentUser.username;
const s3Id = rootState.templates.currentVideo.stock_s3_id;
const type = rootState.dataClay.fileFormat || state.type;
const vid = new Whammy.fromImageArray(state.captures, 30);
vid.lastModifiedDate = new Date();
vid.name = "canvasVideo.webm";
const data = new FormData();
const id = `${username}_${new Date().getTime()}`;
data.append("id", id);
data.append("upload", vid);
const projectId = await dispatch(INSERT_PROJECT);
await dispatch(UPLOAD_TEMP_FILE, data);
const key = await dispatch(CONVERT_FILE_TYPE, { id, username, type, projectId});
const role = rootState.user.currentUser.role;
state.file = `/api/files/${key}`;
let message;
if(role!='banner'){
message =`<p>Your video is ready.</p> Download`;
} else {
message = `<p>Your video is ready. You may download your file from your banner account</p>`;
const resolution = rootState.dataClay.matrix[0];
await dispatch(EXPORT_TO_BANNER, { s3Id, fileUrl: key, extension: `.${type}`, resolution});
}
And here are the api routes called in the actions.
async [UPLOAD_TEMP_FILE]({ commit }, data) {
try {
const response = await axios.post("/api/canvas-editor/upload-temp", data);
return response.data;
} catch (error) {
console.log(error);
}
},
async [CONVERT_FILE_TYPE]({commit}, data) {
try{
const response = await axios.post("/api/canvas-editor/ffmpeg", data);
return response.data;
} catch(error){
console.log(error);
}
}
}
As I said all my routes work and the application runs as expected on localhost however when uploaded to aws I receive unexpected errors.
After some digging I found out that I did not set the ffmpeg path.
Once this was done it worked great.
const ffmpeg = require('fluent-ffmpeg');
const ffmpegPath = require('#ffmpeg-installer/ffmpeg').path;
ffmpeg.setFfmpegPath(ffmpegPath);
module.exports = ffmpeg;

Resources