Creating a 2D Snapshot using three JS library inside Node JS - node.js

Question

I'm using the below code to create 3D model using the Three JS library. Code was running successfully in node JS, but when I'm trying to save an image, I'm getting output as a blank image.
I do have a ThreeUtil JS file which is holding the constants of my shader materials and a couple of functions. There is no problem with this utility file
Observation: I see my buffer data with all element values as zero itself. Can someone let me know where exactly I'm going wrong with my implementation?
Please find the code in below sandbox URL
https://codesandbox.io/s/boring-lewin-7msxm
Attached sample input file as sampleinput.json . from an endpoint need to class main typescript file which will perform the creation and capturing screenshot
import Jimp from 'jimp';
const XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
const THREE = require('three');
const THREESTLLoader = require('three-stl-loader');
const THREEObjLoader = require('three-obj-loader');
const jsdom = require('jsdom');
const gl = require('gl');
import { WebGLRenderTarget } from 'three';
const HOMUNCULUS_OBJ_URL = './assets/3d-homunculus.obj';
const Canvas = require('canvas');
export class ThreeDimensionalModel {
private prostateMeshData: ProstateMeshData;
private width: number = 660;
private height: number = 660;
private dimensions: ThreeCanvasDimensions;
private scenes: Scenes;
private camera: THREE.PerspectiveCamera;
private matrix: THREE.Matrix4;
private rotationMatrix: THREE.Matrix4;
private prostateMeshBoundingBox: THREE.Box3;
private renderer: THREE.WebGLRenderer;
constructor() {
const { JSDOM } = jsdom;
const { window } = new JSDOM();
const { document } = (new JSDOM('')).window;
// #ts-ignore
global.window = window;
// #ts-ignore
global.document = document;
// #ts-ignore
global.window.decodeURIComponent = global.decodeURIComponent;
// Adding XML HTTP Requests library
// #ts-ignore
global.XMLHttpRequest = XMLHttpRequest;
this.dimensions = this.calculateDimensions();
this.scenes = this.initializeScenes();
this.camera = this.initializeCamera();
this.decorateScenes();
this.loadModels(this.prostateMeshData);
this.renderer = this.createRenderer();
this.animate();
}
private calculateDimensions = (): ThreeCanvasDimensions => ({
width: this.width,
height: this.height,
aspect: this.width / this.height,
})
private initializeScenes = (): Scenes => ({
main: new THREE.Scene(),
homunculus: new THREE.Scene(),
})
private initializeCamera(): THREE.PerspectiveCamera {
const camera = new THREE.PerspectiveCamera(
70,
this.dimensions.aspect,
1,
10000);
camera.position.z = 100;
return camera;
}
private decorateScenes(): void {
const ambientLight = new THREE.AmbientLight(0x505050);
const spotLight = this.getSpotLight();
const plane = this.getPlane();
ThreeUtil.addObjectToAllScenes(this.scenes, spotLight);
ThreeUtil.addObjectToAllScenes(this.scenes, ambientLight);
ThreeUtil.addObjectToAllScenes(this.scenes, plane);
}
private loadModels(prostateMeshData: ProstateMeshData): void {
const loader = new THREESTLLoader(THREE);
const dataUri = `data:text/plain;base64,${prostateMeshData.prostateMeshUrl}`;
loader.prototype.load(dataUri, (geometry: THREE.Geometry) => {
this.setModelMatricesAndProstateMeshBoundings(geometry);
this.setupHomunculusMesh();
const prostateMesh = this.createProstateMesh(geometry);
let biopsyMeshes;
if (prostateMeshData.biopsies) {
biopsyMeshes = this.createBiopsyMeshes(prostateMeshData.biopsies);
}
if (prostateMeshData.rois) {
this.loadRoiMeshesOnScene(prostateMeshData.rois);
}
this.addToMainScene(prostateMesh);
if (prostateMeshData.biopsies) {
biopsyMeshes.forEach(biopsyMesh => this.addToMainScene(biopsyMesh));
}
this.scaleInView();
});
}
public saveAsImage() {
// after init of renderer
const renderTarget = new WebGLRenderTarget(this.width, this.height);
const target = this.renderer.getRenderTarget() || renderTarget;
const size = this.renderer.getSize();
// #ts-ignore
target.setSize(size.width, size.height);
this.renderer.setRenderTarget(renderTarget);
this.renderer.render( this.scenes.main, this.camera, target );
// retrieve buffer from render target
const buffer = new Uint8Array(this.width * this.height * 4);
this.renderer.readRenderTargetPixels(renderTarget, 0, 0, this.width, this.height, buffer);
// create jimp image
const image = new Jimp({
data: buffer,
width: this.width,
height: this.height
}, (err: any, images: any) => {
images.write('output.jpeg');
});
}
private createRenderer(): THREE.WebGLRenderer {
const canvasWidth = 1024;
const canvasHeight = 1024;
const canvas = new Canvas.Canvas(canvasWidth, canvasHeight);
canvas.addEventListener = () => {
// noop
}
canvas.style = {};
const renderer = new THREE.WebGLRenderer({ context: gl(this.width, this.height), canvas , preserveDrawingBuffer: true , alpha: true});
renderer.autoClear = false;
renderer.setClearColor(0xFFFFFF);
renderer.setPixelRatio(2);
renderer.setSize(this.dimensions.width, this.dimensions.height);
renderer.sortObjects = false;
renderer.shadowMap.enabled = true;
renderer.shadowMap.type = THREE.PCFShadowMap;
return renderer;
}
private getPlane = (): THREE.Mesh => new THREE.Mesh(
new THREE.PlaneBufferGeometry(2000, 2000, 8, 8),
new THREE.MeshBasicMaterial({ visible: false })
)
private animate = (): void => {
this.renderScenes();
}
private getSpotLight(): THREE.SpotLight {
const spotLight = new THREE.SpotLight(0xffffff, 1.5);
spotLight.position.set(0, 500, 2000);
spotLight.castShadow = true;
spotLight.shadow.camera.near = 200;
spotLight.shadow.camera.far = this.camera.far;
spotLight.shadow.camera.fov = 50;
spotLight.shadow.bias = -0.00022;
spotLight.shadow.mapSize.width = 2048;
spotLight.shadow.mapSize.height = 2048;
return spotLight;
}
private setModelMatricesAndProstateMeshBoundings(geometry: THREE.Geometry): void {
geometry.computeFaceNormals();
geometry.computeBoundingBox();
// A transformation matrix (matrix) is applied to prostate segmentation, target and biopsy,
// which contains a translation and a rotation
// The translation shifts the center of segmentation mesh to view center
this.prostateMeshBoundingBox = geometry.boundingBox.clone();
const center = this.prostateMeshBoundingBox.getCenter(new THREE.Vector3());
const translationMatrix = new THREE.Matrix4().makeTranslation(-center.x, -center.y, -center.z);
// The default view (DICOM coordinate) is patient head pointing towards user.
// We like patient foot pointing towards user.
// Therefore a 180 degree rotation along x (left-right) axis is applied.
const rotAxis = new THREE.Vector3(1, 0, 0);
this.rotationMatrix = new THREE.Matrix4().makeRotationAxis(rotAxis, Math.PI);
// matrix is the multiplication of rotation and translation
this.matrix = new THREE.Matrix4().multiplyMatrices(this.rotationMatrix, translationMatrix);
}
private createProstateMesh(geometry: THREE.Geometry): THREE.Mesh {
const shaderMaterial = ThreeUtil.createShaderMaterial(new THREE.Vector4(0.8, 0.6, 0.2, 0));
geometry.computeFaceNormals();
geometry.computeBoundingBox();
// A transformation matrix (matrix) is applied to prostate segmentation, target and biopsy,
// which contains a translation and a rotation
// The translation shifts the center of segmentation mesh to view center
this.prostateMeshBoundingBox = geometry.boundingBox.clone();
const mesh = new THREE.Mesh(geometry, shaderMaterial);
mesh.applyMatrix(this.matrix);
mesh.name = 'prostate';
return mesh;
}
private renderScenes(): void {
this.renderer.clear();
this.renderer.setViewport(-150, 0, 210, 75);
this.renderer.render(this.scenes.homunculus, this.camera);
this.renderer.clearDepth();
this.renderer.setViewport(0, 0, this.dimensions.width, this.dimensions.height);
this.renderer.render(this.scenes.main, this.camera);
}
private toCameraCoordinates(position: THREE.Vector3): THREE.Vector3 {
return position.applyMatrix4(this.camera.matrixWorldInverse);
}
private scaleInView(): void {
const point1 = this.toCameraCoordinates(this.prostateMeshBoundingBox.min);
const point2 = this.toCameraCoordinates(this.prostateMeshBoundingBox.max);
this.camera.fov = ThreeUtil.angleInDegree(point1, point2, this.camera) * 2;
this.camera.updateProjectionMatrix();
}
private createBiopsyMeshes(biopsiesMeshData: BiopsyMeshData[]): THREE.Mesh[] {
return biopsiesMeshData.map((biopsyMeshData: BiopsyMeshData, index: number) => {
const startPoint = new THREE.Vector3(
biopsyMeshData.proximalX,
biopsyMeshData.proximalY,
biopsyMeshData.proximalZ
);
const endPoint = new THREE.Vector3(biopsyMeshData.distalX, biopsyMeshData.distalY, biopsyMeshData.distalZ);
const color = biopsyMeshData.adenocarcinoma ?
0x0000ff: 0x002233;
const opacity = biopsyMeshData.adenocarcinoma ? 0.65 : 0.3;
const material = new THREE.MeshLambertMaterial({
transparent: true,
depthWrite: false,
depthTest: false,
emissive: new THREE.Color(color),
opacity,
});
const cylinder = ThreeUtil.getCylinderBetweenPoints(startPoint, endPoint, material);
cylinder.applyMatrix(this.matrix);
cylinder.name = `biopsy_${index}`;
return cylinder;
});
}
private loadRoiMeshesOnScene(roisMeshData: RoiMeshData[]): void {
roisMeshData.forEach((roiMeshData: RoiMeshData, index: number) => this.setupRoiMesh(roiMeshData, index));
}
private setupRoiMesh(roiMeshData: RoiMeshData, index: number): void {
const loader = new THREESTLLoader(THREE);
const dataUri = `data:text/plain;base64,${roiMeshData.ROIMeshUrl}`;
if (roiMeshData.tooltipConfiguration) {
if (typeof(roiMeshData.tooltipConfiguration) === 'string' ) {
roiMeshData.tooltipConfiguration = JSON.parse(roiMeshData.tooltipConfiguration);
}
}
loader.prototype.load(dataUri, (roiGeometry: THREE.Geometry) => {
const color = new THREE.Color(0xA396CB);
const colorVector = new THREE.Vector4(color.r, color.g, color.b, 0);
const shaderMaterial = ThreeUtil.createShaderMaterial(colorVector, roiMeshData.adenocarcinoma);
const mesh = new THREE.Mesh(roiGeometry, shaderMaterial);
mesh.name = `roi_${roiMeshData.ROIId}`;
roiGeometry.applyMatrix(this.matrix);
this.addToMainScene(mesh);
this.saveAsImage();
});
}
private setupHomunculusMesh(): void {
new THREEObjLoader(THREE);
new THREE.OBJLoader().load(HOMUNCULUS_OBJ_URL, (group: THREE.Group) => {
group.traverse((child) => {
if (child instanceof THREE.Mesh) {
// apply custom material
// #ts-ignore
child.material = new THREE.MeshPhongMaterial({
color: 0xb09b70,
specular: 0x050505,
shininess: 100,
flatShading: false
});
const homunculusMatrix = new THREE.Matrix4();
homunculusMatrix.makeScale(0.020, 0.020, 0.020);
homunculusMatrix.multiply(this.rotationMatrix);
child.applyMatrix(homunculusMatrix);
}
});
this.scenes.homunculus.add(group);
});
}
private addToMainScene(object3D: THREE.Object3D): void {
this.scenes.main.add(object3D);
// Adding Ambient Light for all scenes inside main scenes except Humanoid
const ambientLight = new THREE.AmbientLight(0x000000);
this.scenes.main.add(ambientLight);
}
}