Playing with node steams
This code reads from index.js and writes to indexCopy.js - kind of file copy.
Target file got created, but during execution exception is thrown:
node index.js
events.js:183
throw er; // Unhandled 'error' event
^
Error: stream.push() after EOF
at readableAddChunk (_stream_readable.js:240:30)
at MyStream.Readable.push (_stream_readable.js:208:10)
at ReadStream.f.on (C:\Node\index.js:16:28)
at emitOne (events.js:116:13)
at ReadStream.emit (events.js:211:7)
at addChunk (_stream_readable.js:263:12)
at readableAddChunk (_stream_readable.js:250:11)
at ReadStream.Readable.push (_stream_readable.js:208:10)
at fs.read (fs.js:2042:12)
at FSReqWrap.wrapper [as oncomplete] (fs.js:658:17)
C:\Node>
This is code:
var util = require('util');
var stream = require('stream');
var fs = require('fs');
var MyStream = function(){
stream.Readable.call(this)
}
util.inherits(MyStream,stream.Readable);
MyStream.prototype._read = function(d){
f = fs.createReadStream("index.js");
f.on('data',(d)=>{this.push(d)});
f.on('end',()=>{this.push(null)}); //when file finished need to close stream
}
var f = fs.createWriteStream("indexCopy.js")
var myStream = new MyStream()
myStream.pipe(f);
I tried to call this.push(null) in 'data' event, in that case even target file is not created and code fails with the exception.
I realize that copy file should be done easier with pipe() function - I am just experimenting/learning.
What is wrong with my approach?
You don't want the f = fs.createReadStream("index.js") line inside the _read method -- _read gets called repeatedly so you're creating multiple read streams. Put that in your constructor instead.
function MyStream () {
stream.Readable.call(this);
this.source = fs.createReadStream("index.js");
this.haveBound = false;
}
MyStream.prototype._read = function () {
if (this.haveBound) return; // Don't bind to events repeatedly
this.haveBound = true;
this.source.on("data", d => this.push(d));
this.source.on("end", () => this.push(null));
};
This is awkward though. Streams are meant to be pipe'ed.
Related
I have a problem with nodejs when trying to write to stdin of a process I have spawned with this spawn function of child_process
const spawn = require("child_process").spawn;
class Barotrauma {
static instance = null;
server = null;
constructor() {
this.server = spawn(
"F:\\dev\\barotrauma\\steamcmd\\steamapps\\common\\Barotrauma Dedicated Server\\DedicatedServer.exe",
{
stdio: [process.stdin, process.stdout, process.stderr],
}
);
}
static getInstance() {
if (Barotrauma.instance === null) {
Barotrauma.instance = new Barotrauma();
} else {
return Barotrauma.instance;
}
}
sendCommand(command) {
//this.server.stdout.write(`${command}\n`)
// this is a test to get an output on command execution to see if it works
process.stdin.write("help\n");
}
}
module.exports = Barotrauma;
So, this snippet of code is for starting a game server then send command to it on socket events (the socket call sendCommand function)
if I try to write commands in the console it works fine, but if I try to execute the sendCommand function it crash with error :
node:events:498
throw er; // Unhandled 'error' event
^
Error: write EPIPE
at afterWriteDispatched (node:internal/stream_base_commons:160:15)
at writeGeneric (node:internal/stream_base_commons:151:3)
at ReadStream.Socket._writeGeneric (node:net:795:11)
at ReadStream.Socket._write (node:net:807:8)
at writeOrBuffer (node:internal/streams/writable:389:12)
at _write (node:internal/streams/writable:330:10)
at ReadStream.Writable.write (node:internal/streams/writable:334:10)
at Barotrauma.sendCommand (F:\dev\barotrauma\serverManager.js:26:19)
at handleReward (F:\dev\barotrauma\handler.js:4:28)
at WebSocket.connection.onmessage (F:\dev\barotrauma\index.js:26:5)
Emitted 'error' event on ReadStream instance at:
at emitErrorNT (node:internal/streams/destroy:157:8)
at emitErrorCloseNT (node:internal/streams/destroy:122:3)
at processTicksAndRejections (node:internal/process/task_queues:83:21) {
errno: -4047,
code: 'EPIPE',
syscall: 'write'
}
The crash seems to originate from the process.stin.write function.
Any idea how to solve this?
You might want to refer to options.stdio.
Without further depth into the full context it's hard to say what's exactly going on, but the following might work as you'd expect:
If you really need the parent and children fds to be connected you could use pipe for the stdio option and have data be propagated between the two processes.
The following snippet (reduced to minimize noise) should do the trick:
class Barotrauma {
server = null;
constructor() {
// Default `stdio` option is `pipe` for fds 0, 1 and 2 (stdio, stdout and stderr)
this.server = spawn(
"F:\\dev\\barotrauma\\steamcmd\\steamapps\\common\\Barotrauma Dedicated Server\\DedicatedServer.exe"
);
// Catches incoming messages from child, if necessary
this.server.stdout.on('data', (d) => {
console.log(`Message from child: '${d}'`)
})
}
sendCommand(cmd) {
// Sends messages to child
this.server.stdin.write(`${cmd}\n`);
}
}
Given you're in a Windows environment, mind you might want to use overlapped instead of pipe if you need async I/O to communicate with that application. For more info, refer to the official docs.
I'm trying to onboard a new developer, that is using Windows 11 as the only one on our small team. I've guided him through installing WSL2 and Ubuntu 20.04.3 LTS (linux kernel: 5.10.93.2-microsoft-standard-WSL2).
We are 3 other developers who are using native Ubuntu, WSL2 Ubuntu 21.04 and macOS, respectively.
We are all on nodejs 16.14 with the exact same package-lock.json file.
He is the only one getting an Error [ERR_STREAM_WRITE_AFTER_END]: write after end.
tldr;
Both errors are about writing to a Buffer.
Is anyone aware of any related issues to nodejs's Buffer implementation on Windows 11?
We are using thinky as outlined below:
'use strict';
const createThinky = require('thinky');
const { rethinkdbConfig } = require ('./utils/config.js');
const thinky = createThinky (rethinkdbConfig);
// Thinky is our ORM
var type = thinky.type;
// Creates the thinky DB model for payments - we save the amount and the stripe customerID for each transaction
// You can execute any rethinkdb query language on the model, eg Payment.count().execute()
var Payment = thinky.createModel("payments", {
id: type.string(),
amount: type.number(),
customerID: type.string(),
project: type.string(),
projectName: type.string(),
projectPercentage: type.number(),
firefundPercentage: type.number(),
type: type.string(),
processor: type.string(),
email: type.string(),
charged: type.boolean(),
recharged: type.boolean()
});
module.exports = {
model: Payment,
};
Now, it doesn't matter if he connects to a local instance of rethinkdb (2.4.1~0focal), our staging or production rethinkdb (2.3.5~0trusty) on AWS.
I got him to try Netcat with nc -zv [url] 28015 to see if he could connect at all and he got connectıon successful. So I do not think it is a firewall issue.
Error stack trace
node ./bin/www
firefund:www Listening on port 3000 +0ms
node:events:498
throw er; // Unhandled 'error' event
^
Error [ERR_STREAM_WRITE_AFTER_END]: write after end
at new NodeError (node:internal/errors:371:5)
at _write (node:internal/streams/writable:319:11)
at Socket.Writable.write (node:internal/streams/writable:334:10)
at Connection._sendProof (/home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/connection.js:294:19)
at /home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/connection.js:248:12
at Object.tryCatch (/home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/helper.js:170:3)
at Connection._computeSaltedPassword (/home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/connection.js:247:12)
at Socket.<anonymous> (/home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/connection.js:184:18)
at Socket.emit (node:events:520:28)
at Socket.emit (node:domain:475:12)
at addChunk (node:internal/streams/readable:315:12)
at readableAddChunk (node:internal/streams/readable:289:9)
at Socket.Readable.push (node:internal/streams/readable:228:10)
at TCP.onStreamRead (node:internal/stream_base_commons:190:23)
Emitted 'error' event on Connection instance at:
at Socket.<anonymous> (/home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/connection.js:129:12)
at Socket.emit (node:events:520:28)
at Socket.emit (node:domain:475:12)
at emitErrorNT (node:internal/streams/destroy:157:8)
at emitErrorCloseNT (node:internal/streams/destroy:122:3)
at processTicksAndRejections (node:internal/process/task_queues:83:21) {
code: 'ERR_STREAM_WRITE_AFTER_END'
}
The stack trace indicates that it is a connection error in node_modules/rethinkdbdash/lib/connection.js.
connection.js
connection.js: line 294 is this.connection.write(Buffer.concat([new Buffer(message.toString()), NULL_BUFFER])) at the bottom.
Connection.prototype._sendProof = function(authentication, randomNonce, saltedPassword) {
var clientFinalMessageWithoutProof = "c=biws,r=" + randomNonce;
var clientKey = crypto.createHmac("sha256", saltedPassword).update("Client Key").digest()
var storedKey = crypto.createHash("sha256").update(clientKey).digest()
var authMessage =
"n=" + this.user + ",r=" + this.randomString + "," +
authentication + "," +
clientFinalMessageWithoutProof
var clientSignature = crypto.createHmac("sha256", storedKey).update(authMessage).digest()
var clientProof = helper.xorBuffer(clientKey, clientSignature)
var serverKey = crypto.createHmac("sha256", saltedPassword).update("Server Key").digest()
this.serverSignature = crypto.createHmac("sha256", serverKey).update(authMessage).digest()
this.state = 2
var message = JSON.stringify({
authentication: clientFinalMessageWithoutProof + ",p=" + clientProof.toString("base64")
})
this.connection.write(Buffer.concat([new Buffer(message.toString()), NULL_BUFFER]))
}
Variant Error
He has also reported a variant of the same error as posted below:
node ./bin/www
firefund:www Listening on port 3000 +0ms
node:events:498
throw er; // Unhandled 'error' event
^
Error [ERR_STREAM_WRITE_AFTER_END]: write after end
at new NodeError (node:internal/errors:371:5)
at _write (node:internal/streams/writable:319:11)
at Socket.Writable.write (node:internal/streams/writable:334:10)
at /home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/connection.js:143:23
at Object.tryCatch (/home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/helper.js:170:3)
at Socket.<anonymous> (/home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/connection.js:142:12)
at Socket.emit (node:events:523:35)
at Socket.emit (node:domain:475:12)
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1143:10)
Emitted 'error' event on Connection instance at:
at Socket.<anonymous> (/home/edel_weiss/firefund-production/node_modules/rethinkdbdash/lib/connection.js:129:12)
at Socket.emit (node:events:520:28)
at Socket.emit (node:domain:475:12)
at emitErrorNT (node:internal/streams/destroy:157:8)
at emitErrorCloseNT (node:internal/streams/destroy:122:3)
at processTicksAndRejections (node:internal/process/task_queues:83:21) {
code: 'ERR_STREAM_WRITE_AFTER_END'
}
This variant suggest that the error is at line 143 in connection.js:
self.connection.write(Buffer.concat([versionBuffer, authBuffer, NULL_BUFFER]));
self.connection.on('connect', function() {
self.connection.removeAllListeners('error');
self.connection.on('error', function(error) {
self.emit('error', error);
});
var versionBuffer = new Buffer(4)
versionBuffer.writeUInt32LE(protodef.VersionDummy.Version.V1_0, 0)
self.randomString = new Buffer(crypto.randomBytes(18)).toString('base64')
var authBuffer = new Buffer(JSON.stringify({
protocol_version: PROTOCOL_VERSION,
authentication_method: AUTHENTIFICATION_METHOD,
authentication: "n,,n=" + self.user + ",r=" + self.randomString
}));
helper.tryCatch(function() {
self.connection.write(Buffer.concat([versionBuffer, authBuffer, NULL_BUFFER]));
}, function(err) {
// The TCP connection is open, but the ReQL connection wasn't established.
// We can just abort the whole thing
self.open = false;
reject(new Err.ReqlDriverError('Failed to perform handshake with '+self.host+':'+self.port).setOperational());
});
});
If you read this far - THANK YOU!
Both errors are about writing to a Buffer.
Is anyone aware of any related issues to nodejs's Buffer implementation on Windows 11?
Definitely not sure if this is THE answer, just spit-balling here that maybe you could make a custom Buffer.concat function to work in case it really is the nodejs Buffer.concat being the cause of the problem here ;-; hope it works I'm on windows 10 however
function BufferConcat(buffers){ //buffers is the array of buffers
var bytes=Buffer.byteLength, i=0
var length=buffers.reduce((b1,b2)=>bytes(b1)+bytes(b2))
const BufferToReturn=Buffer.alloc(length)
for(let buffer in buffers){
let values=Object.values(buffer)
for(let item of buffer){BufferToReturn[i++]=item}
}
return BufferToReturn
}
When I execute a command (the error doesn't occur when I type a normal message) in my discord bot I get an error saying:
Cannot read property 'active' of undefined
And it occurs when I try to console log an object from a json file where I store users data.
Those are the first lines of code of the index.js file of my bot, the line where I try to console log is where the error occurs
const Discord = require('discord.js');
const client = new Discord.Client();
const fs = require("fs");
const prefix = '>';
let xp = require("./storage/dbGeneral.json");
let pr = require("./storage/dbPremium.json");
let lv = require("./storage/levels.json");
client.on('message', message => {
console.log(pr[message.author.id].active);
}
This is the json file where I store the data
{
"397387465024864257": {
"active": false,
"dateStart": "",
"dateEnd": ""
}
}
This is the error:
index.js:14
console.log(pr[message.author.id].active);
^
TypeError: Cannot read property 'active' of undefined
at Client.client.on.message (index.js:14:39)
at emitOne (events.js:116:13)
at Client.emit (events.js:211:7)
at MessageCreateHandler.handle (\node_modules\discord.js\src\client\websocket\packets\handlers\MessageCreate.js:9:34)
at WebSocketPacketManager.handle (\node_modules\discord.js\src\client\websocket\packets\WebSocketPacketManager.js:103:65)
at WebSocketConnection.onPacket (\node_modules\discord.js\src\client\websocket\WebSocketConnection.js:333:35)
at WebSocketConnection.onMessage (\node_modules\discord.js\src\client\websocket\WebSocketConnection.js:296:17)
at WebSocket.onMessage (\node_modules\ws\lib\event-target.js:120:16)
at emitOne (events.js:116:13)
at WebSocket.emit (events.js:211:7)
I really don't know what is the cause of the error, all other json requests work fine with the same method.
The error is caused by trying to access a property of something not defined.
To fix your problem, you must check if the user exists in your database.
let user = pr[message.author.id];
if(user) console.log(user.active)
Using Polling like below to check if the content of the file is changed then, other two functions are called
var poll_max_date=AsyncPolling(function (end,err) { if(err) {
console.error(err); } var stmp_node_id=fs.readFileSync(path.join(__dirname,'node_id'),"utf8");
console.log("--------loaded node : "+stmp_node_id);
if(druid_stmp_node_id!=stmp_node_id) {
// MAX DATA CUT-OFF DRUID QUERY
druid_exe.max_date_query_fire();
// // DRUID QUERY FOR GLOBAL DATA
druid_exe.global_druid_query_fire();
druid_stmp_node_id=stmp_node_id; }
end(); }, 1800000).run();//30 mins
Its working fine for sometime, but then getting below error like after 4 - 5hours :
events.js:167
throw er; // Unhandled 'error' event
^
Error: read ECONNRESET
at TCP.onStreamRead (internal/stream_base_commons.js:111:27) Emitted 'error' event at:
at emitErrorNT (internal/streams/destroy.js:82:8)
at emitErrorAndCloseNT (internal/streams/destroy.js:50:3)
tried using fs.watch to monitor the changes in the file instead of polling like below :
let md5Previous = null; let fsWait = false;
fs.watch(dataSourceLogFile, (event, filename) => { if (filename) {
if (fsWait) return;
fsWait = setTimeout(() => {
fsWait = false;
}, 1000);
const md5Current = md5(fs.readFileSync(dataSourceLogFile));
if (md5Current === md5Previous) {
return;
}
md5Previous = md5Current;
console.log(`${filename} file Changed`);
// MAX DATA CUT-OFF DRUID QUERY
druid_exe.max_date_query_fire();
// DRUID QUERY FOR GLOBAL DATA
druid_exe.global_druid_query_fire(); } });
Its is also working fine for sometime, but then getting same error like after 4 - 5hours :
events.js:167 throw er; // Unhandled 'error' event ^
Error: read ECONNRESET at TCP.onStreamRead
(internal/stream_base_commons.js:111:27) Emitted 'error' event at: at
emitErrorNT (internal/streams/destroy.js:82:8) at emitErrorAndCloseNT
(internal/streams/destroy.js:50:3)
But when run in Local Machine, its working fine. the error occurs only when run in remote Linux Machine.
somebody can help me how I can fix that problem?
Use fs.watchFile once , because fs.watch is not consistent across platforms,
https://nodejs.org/docs/latest/api/fs.html#fs_fs_watchfile_filename_options_listener
Change your code according to the requirement.
It has been happening since the users are closing the browser before the data request is received, leading to Connection Reset.
Used PM2 (http://pm2.keymetrics.io/) to run the application, and it is working great now .
I made following, to play a bit around with node.js.
The files in the folder zipfiles are zipped accordingly and everything seems to work.
But I got an error on the cmd and I don't know where it comes from or how to solve it.
events.js:72
throw er; // Unhandled 'error' event
^
Error: write after end
at writeAfterEnd (_stream_writable.js:130:12)
at Gzip.Writable.write (_stream_writable.js:178:5)
at write (_stream_readable.js:583:24)
at flow (_stream_readable.js:592:7)
at ReadStream.pipeOnReadable (_stream_readable.js:624:5)
at ReadStream.EventEmitter.emit (events.js:92:17)
at emitReadable_ (_stream_readable.js:408:10)
at emitReadable (_stream_readable.js:404:5)
at readableAddChunk (_stream_readable.js:165:9)
at ReadStream.Readable.push (_stream_readable.js:127:10)
Here's my script:
var zlib = require('zlib');
var gzip = zlib.createGzip();
var fs = require('fs');
var zip = {
zipAll: function(dir){
//files to zip
fs.readdir(dir, function(err, data){
if(err) throw(err);
var arrayValue = data.toString().split(',');
//files with .gz at the end, needs to be excluded
for(var i=0; i<arrayValue.length; i+=1){
console.log("Zipping following files: " + arrayValue[i]);
var input = fs.createReadStream('zipfiles/' + arrayValue[i]);
var output = fs.createWriteStream('zipfiles/input'+[i]+'.txt'+'.gz');
input.pipe(gzip).pipe(output);
}
});
}
};
zip.zipAll('zipfiles');
Thanks
The Gzip object is a bit wonky (afaik undocumented) to reuse for multiple files. The easiest way to fix your problem is to simply use a separate gzip object per file to compress, something like;
for(var i=0; i<arrayValue.length; i+=1){
console.log("Zipping following files: " + arrayValue[i]);
var input = fs.createReadStream('zipfiles/' + arrayValue[i]);
var output = fs.createWriteStream('zipfiles/input'+[i]+'.txt'+'.gz');
input.pipe(zlib.createGzip()).pipe(output);
}