All,
I have an array of sizes such as sizes = [20,40,60,80]
I need to loop over those, resize the src image to each size and name them appropriately.
Then, using knox, upload them to s3, and then remove the resized image.
Here is what I have:
http://jsfiddle.net/bpoppa/ZAcA7/
The problem is the flow control. When knox tries to putFile, I get an error saying it doesn't exist, which means knox is either running before the resize is done or it is already deleted at that point.
Any tips? All help is appreciated.
Thanks!
You need to remember that node.js code runs asynchronously. In your original code, the knox code is running before image.resize has completed (the callback is used to tell you when the operation has completed, not just to handle errors). Node won't wait for the callback and just continues to execute code in your function. You also need to be careful of using anonymous callbacks in for loops without creating a closure.
In general you want to use the callbacks to control program flow like the code below so that you only do the following action when the preceding action has completed.
var src = name + '.jpg';
for (var i = sizes.length - 1; i >= 0; i--) {
var k = i;
var dest = sizes[k] + '.jpg';
var s3 = sizes[k] + '.jpg';
resizeAndPut(src, dest, s3, sizes[k]);
}
fs.unlink(src); /* delete the source file now */
var resizeAndPut = function (src, dest, s3, size) {
easyimage.resize(
{
src: src,
dst: dest,
width: size,
height: size
}, function(err, image) {
if (err) throw err;
knox.putFile(dest, s3, function(err, res) { /* does image contain the path?, if so, might want to use image.path or the like instead of dest here */
if (err) throw err;
fs.unlink(dest); /* delete the local file*/
});
});
};
Related
I am using the sharp nodejs library found here: https://github.com/lovell/sharp. I am trying to take several screenshots, and then piece the images together using the sharp library.
Here is my code. I am using puppeteer to take screenshots of the page, saving in memory as a binary file and combining those binary files together using sharp's composite() method.
let pagePath = 'path/to/file.png';
let maxScreenshotHeight = 2000;
// Loop over sections of the screen that are of size maxScreenshotHeight.
for (let ypos = 0; ypos < contentSize.height; ypos += maxScreenshotHeight) {
const height = Math.min(contentSize.height - ypos, maxScreenshotHeight);
let image = await page.screenshot({
encoding: 'binary',
clip: {
x: 0,
y: ypos,
width: contentSize.width,
height
}
});
composites.push({input: image, gravity: 'southeast'});
}
sharp()
.composite(composites)
.toFile(pagePath, function(err) {
if (err) {
console.log('fail');
return;
}
console.log('complete');
});
However, in the toFile callback, nothing ever gets logged. Console logging works, as I've added logs before and after the toFile statement, but it seems that this function call never completes. I want to create a png file that I can later download.
How can I merge these multiple screenshots and store them on the server for a later download? Am I using toFile incorrectly?
I'm using Jimp module of NodeJS to do some image transformation in my (big) nodejs script inside a big loop where my filename is dynamically generated.
When doing my jimp image transformation, the filename provided to write function parameter is already changed. Because the write is inside a callback (so a another thread?) and my loop is already continue to process another loop step.
Here is an example overview:
for (var i = 0; i < 10; ++i) {
var filename = 'test' + i + '.png';
//some script to generate my image : pupperteer screenshot of a webpage
Jimp.read(filename).then(function (image) {
image.greyscale().write(filename);
}).catch(function (err) {
console.error(err);
});
}
In this example, my script create the file test1.png in color, then I can view a test2.png appear which is a copy of test1.png but in greyscale... Then it overwrite by a new color image named test2.png.
So I'm wondering how to solve this?
It's perfectly fine to have it in multithread, so how to use a copy of "filename" string to use it in parameter of write function?
Regards
Alex
use let instead of var
let filename = 'test' + i + '.png';
You have a problem with asynchrony. I think that you could create a method and send filename as param. For example:
for (var i = 0; i < 10; ++i) {
let filename = 'test' + i + '.png';
//some script to generate my image : pupperteer screenshot of a webpage
_saveImg(filename);
}
function _saveImg(name) {
const filename = name;
Jimp.read(filename).then(function (image) {
image.greyscale().write(filename);
}).catch(function (err) {
console.error(err);
});
}
=)
I'm kinda new to NodeJS and I'm working on a simple file encoder.
I planned to change the very first 20kb of a file and just copy the rest of it.
So I used the following code, but it changed some bytes in the rest of the file.
Here is my code:
var fs = require('fs');
var config = require('./config');
fs.open(config.encodeOutput, 'w', function(err, fw) {
if(err) {
console.log(err);
} else {
fs.readFile(config.source, function(err, data) {
var start = 0;
var buff = readChunk(data, start);
while(buff.length) {
if(start < config.encodeSize) {
var buffer = makeSomeChanges(buff);
writeChunk(fw, buffer);
} else {
writeChunk(fw, buff);
}
start += config.ENCODE_BUFFER_SIZE;
buff = readChunk(data, start);
}
});
}
});
function readChunk(buffer, start) {
return buffer.slice(start, start + config.ENCODE_BUFFER_SIZE);
}
function writeChunk(fd, chunk) {
fs.writeFile(fd, chunk, {encoding: 'binary', flag: 'a'});
}
I opened encoded file and compared it with the original file.
I even commented these parts:
//if(start < config.encodeSize) {
// var buffer = makeSomeChanges(buff);
// writeChunk(fw, buffer);
//} else {
writeChunk(fw, buff);
//}
So my program just copies the file, but it still changes some bytes.
What is wrong?
So I checked the pattern and I realized some bytes are not in the right place and I guessed that it should be because I'm using async write function.
I changed fs.writeFile() to fs.writeFileSync() and everything is working fine now.
Since you were using asynchronous IO, you should've been waiting for a queue of operations, as multiple writes happening at the same time are likely to end up corrupting your file. This explains why your issue is solved using synchronous IO — this way, a further write cannot start before the previous one completed.
However, using synchronous APIs when asynchronous ones are available is a poor choice, due to which your program will be actually blocked while it writes to the file. You should go for async and create a queue to wait for.
I need to write a file temporarily to the file system in order to run a quick check on it, and I then want it deleted.
Based on my Googling, it looks like the tmp package for NodeJS can be used:
https://www.npmjs.com/package/tmp
But I'm really confused by the documentation.
This is the example they give on how to use it to create a temporary file:
var tmp = require('tmp');
tmp.file(function _tempFileCreated(err, path, fd, cleanupCallback) {
if (err) throw err;
console.log("File: ", path);
console.log("Filedescriptor: ", fd);
// If we don't need the file anymore we could manually call the cleanupCallback
// But that is not necessary if we didn't pass the keep option because the library
// will clean after itself.
cleanupCallback();
});
But when I read that, it looks like its passing a function into tmp.file. How do I pass a buffer, a path, or something into this for its to do its thing and create the temporary file?
I must be missing something silly.
Thank-you for your help!
------------- Final Answer -------------------------------------------------
Figured I'd post my final answer in case it helped someone else who somehow had a brain block when reading the example code. This should have been obvious now that I see the problem. Thank-you CViejo.:
var fs = require('fs');
var Q = require('q');
var tmp = require('tmp');
self=this;
/**
* writeBufferToFile - Takes a buffer and writes its entire contents to the File Descriptor location
* #param fd - File Decripter
* #param buffer - Buffer
* #returns {*|promise} - true if successful, or err if errors out.
*/
module.exports.writeBufferToFile = function(fd, buffer){
var deferred = Q.defer();
fs.write(fd, buffer, 0, buffer.length, 0, function (err, written, buffer){
if(!written)
deferred.reject(err);
else
deferred.resolve(true);
});
return deferred.promise;
}
/**
* writeBufferToTempFile - Takes a buffer and writes the entire contents to a temp file
* #param buffer - The buffer to write out.
* #returns {*|promise} - The file path of the file if a success, or the error if a failure.
*/
module.exports.writeBufferToTempFile = function(buffer){
var deferred = Q.defer();
tmp.file(function _tempFileCreated(err, path, fd, cleanupCallback) {
if (err)
deferred.reject(err);
else {
self.writeBufferToFile(fd, buffer).then(
function (isWritten) { deferred.fulfill(path); },
function (err) { deferred.reject(err); });
}
});
return deferred.promise;
}
In short, you don't. The point of this module is precisely to guess the system's default temporary directory, generate random file names and handle the file and directory removal for you.
In that sense it's quite opinionated about where to save and using a specific file name, you can only specify a folder name within the system's temporary directory and prefix / postfix options.
About writing contents to the file, you have to handle it yourself:
var fs = require("fs");
var tmp = require("tmp");
tmp.file(function (err, path, fd, cleanup) {
if (err) throw err;
fs.appendFile(path, new Buffer("random buffer"), function (err) {
if (err)
throw err
});
// fs.appendFile(path, "random text");
});
Other modules like temp or temporary work in similar manner.
I am trying to create a PDF file with PDFKit. I insert an image with like this:
var PDFDocument = require('pdfkit');
var doc = new PDFDocument();
doc.image(some_image_as_buffer);
and it is working like expected. But now want the image be trimmed and I found GraphicsMagick for node.js. But the problem that I have is to make it work with PDFKit. doc.image expects a filename or a buffer, but since I already have a buffer I want to work with buffers (there is no file anywhere because the buffer comes directly from the database).
The trimming works like this:
var gm = require('gm');
gm(some_image_as_buffer, 'image.png')
.trim()
.toBuffer(function(err, trimmed_image_buffer) {
// trimmed_image_buffer is correct,
// but I can't put it to the document like this:
doc.image(trimmed_image_buffer);
// beacause I don't know which page and/or position
// the doc is currently on, because of the asynchronous
// nature of this callback.
});
UPDATE:
For clarification: I want to be able to use the asynchronous trimmed image in the synchronous code for PDFKit. PDFKit only works synchronously and gm doesn't offer a synchronous interface.
UPDATE2:
var gm = require('gm');
gm(some_image_as_buffer, 'image.png')
.trim()
.toBuffer(function(err, trimmed_image_buffer) {
// trimmed_image_buffer is correct,
// but I can't put it to the document like this:
doc.image(trimmed_image_buffer);
// beacause I don't know which page and/or position
// the doc is currently on, because of the asynchronous
// nature of this callback.
});
doc.text('some text');
// is not guaranteed to run after image is inserted
// and a couple of hundred lines more
After the last line in this example there are a lot more lines of code which add content to the PDF, but I don't want to put everything (couple of hundred lines) in one callback just because I need on asynchronous function to manipulate the image.
Is there any way to make this manipulation synchronous?
UPDATE_2
You basically ask for stopping execution of a code until some asynchronous operation has completed. For sure it is not possible in general case.
In case of gm module, it is not possible either. The gm module spawns a new process for executing a command (in your case trim()) and the API for spawning new processes is asynchronous in its very nature.
UPDATE
To make use of promise in your scenario:
var gm = require('gm'),
Q = require('Q'),
PDFDocument = require('pdfkit'),
doc = new PDFDocument();
function getTrimmedImage(some_image_as_buffer){
var deferred = Q.defer();
gm(some_image_as_buffer, 'image.png')
.trim()
.toBuffer(function(err, trimmed_image_buffer) {
if(err) { deferred.reject(err); }
else { deferred.resolve(trimmed_image_buffer); }
});
return deferred.promise;
}
// here goes all manipulations before the trimmed image is inserted
getTrimmedImage(some_image_as_buffer).then(
function(trimmed_image_buffer){
doc.image(trimmed_image_buffer);
// here goes all manipulations after the trimmed image is inserted
}
);
As I wrote in the comment above, a promise based solution should work elegantly. I use Q library, but any other promise library will do the job, as well.
One option would be to collect all resources of asynchronous nature before starting manipulating the pdf. Then you are guaranteed that no race condition occur, though it may slow down the whole process. I used a toy example to have it working in the browser environment, let me know if you have any problems converting it to your use case:
function getAsyncResource(){
var defer = Q.defer();
setTimeout(function(){
var result = "Some value: " + Date.now();
console.log("Async resource resolved: " + result);
defer.resolve(result);
}, Math.random() * 5000);
return defer.promise;
}
function someOperationThatNeedsAsyncResources(A, B, C){
console.log("Have all resources: ", A, B, C);
}
var A = getAsyncResource(),
B = getAsyncResource(),
C = getAsyncResource();
Q.all([A,B,C]).spread(someOperationThatNeedsAsyncResources);
<script src="//cdnjs.cloudflare.com/ajax/libs/q.js/1.1.2/q.js"></script>
Other option would be to split the process into steps, like so:
function getAsyncResource(value){
var defer = Q.defer();
setTimeout(function(){
var result = "Some value: " + value;
console.log("Async resource resolved: " + result);
defer.resolve(result);
}, Math.random() * 5000);
return defer.promise;
}
function nextStep(resource){
console.log("Next step: " + resource);
}
var A = getAsyncResource("A"),
B = getAsyncResource("B"),
C = getAsyncResource("C");
A.then(nextStep)
.then(function(){return B;})
.then(nextStep)
.then(function(){return C;})
.then(nextStep);
<script src="//cdnjs.cloudflare.com/ajax/libs/q.js/1.1.2/q.js"></script>