Node Postgres COPY FROM failing silently - node.js

Question

I am trying to use PostgreSQL's COPY FROM API to stream potentially-thousands of records into a database as they are dynamically generated in node.js code. To do so, I wrote this generic wrapper function:
function streamRows(client, { table, columns, data }) {
return new Promise((resolve, reject) => {
const sqlStream = client.query(
copyFrom(`COPY ${ table } (${ columns.join(', ') }) FROM STDIN`));
const rowStream = new Readable();
rowStream.pipe(sqlStream)
.on('finish', resolve)
.on('error', reject);
for (const row of data) {
rowStream.push(`${ row.join('\t') }\n`);
}
rowStream.push('\\.\n');
rowStream.push(null);
});
}
The database table I'm writing into looks like this:
CREATE TABLE devices (
id SERIAL PRIMARY KEY,
group_id INTEGER REFERENCES groups(id),
serial_number CHAR(12) NOT NULL,
status INTEGER NOT NULL
);
And I am calling it as follows:
function *genRows(id, devices) {
let count = 0;
for (const serial of devices) {
yield [ id, serial, UNSTARTED ];
count++;
if (count % 10 === 0) log.info(`Streamed ${ count } rows...`);
}
log.info(`Streamed ${ count } rows.`);
}
await streamRows(client, {
table: 'devices',
columns: [ 'group_id', 'serial_number', 'status' ],
data: genRows(id, devices),
});
The log statements in my generator function that's producing the per-row data all run as expected, and the output indicates that it is in fact always running the generator to completion, and streaming all the data rows I want. No errors are ever thrown. But if I wait for it to complete, the table sometimes ends up with 0 rows added to it--i.e., it looks like I sent all that data to Postgres, but none of it was actually inserted. What am I doing wrong?

Answer 1

I do not know exactly what parts of this made the difference and what is purely stylistic, but after playing around with a bunch of different examples from across the web, I managed to cobble together this function which works:
function streamRows(client, { table, columns, data }) {
return new Promise((resolve, reject) => {
const iterator = data[Symbol.iterator]();
const rs = new Readable();
const ws = client.query(copyFrom(`COPY ${ table } (${ columns.join(', ') }) FROM STDIN`));
rs._read = function() {
const { value, done } = iterator.next();
rs.push(done ? null : `${ value.join('\t') }\n`);
};
rs.on('error', reject);
ws.on('error', reject);
ws.on('end', resolve);
rs.pipe(ws);
});
}