multiple column copy format postgresql Node.js - node.js

I using postgres stream to insert record into postgres ,
for single column works fine , but what is ideal data format for copy for multiple columns
code snippets
var sqlcopysyntax = 'COPY srt (starttime, endtime) FROM STDIN delimiters E\'\\t\'';
var stream = client.query(copyFrom(sqlcopysyntax));
console.log(sqlcopysyntax)
var interndataset = [
['1', '4'],
['6', '12.074'],
['13.138', '16.183'],
['17.226', '21.605'],
['22.606', '24.733'],
['24.816', '27.027'],
['31.657', '33.617'],
['34.66', '37.204'],
['37.287', '38.58'],
['39.456', '43.669'],
['43.752', '47.297'],
['47.381', '49.55'],
];
var started = false;
var internmap = through2.obj(function(arr, enc, cb) {
/* updated this part by solution provided by #VaoTsun */
var rowText = arr.map(function(item) { return (item.join('\t') + '\n') }).join('')
started = true;
//console.log(rowText)
rowText=rowText+'\\\.';
/* end here*/
started = true;
cb(null, rowText);
})
internmap.write(interndataset);
internmap.end();
internmap.pipe(stream);
wherein i got error: (due to delimiter)missing data for column "endtime"(resolved) but got below error
error: end-of-copy marker corrupt
COPY intern (starttime, endtime) FROM STDIN
1 4
6 12.074
13.138 16.183
17.226 21.605
22.606 24.733
24.816 27.027
31.657 33.617
34.66 37.204
37.287 38.58
39.456 43.669
43.752 47.297
47.381 49.55
any pointer on how to resolve this .
what would be ideal format for multiple column inserts using copy command

With immense help from #jeromew from github community.
and proper implementation of node-pg-copy-streams(takes away copy command complexity ). we were able to solve this issue
https://github.com/brianc/node-pg-copy-streams/issues/65
below is working code snippets
var sqlcopysyntax = 'COPY srt (starttime, endtime) FROM STDIN ;
var stream = client.query(copyFrom(sqlcopysyntax));
console.log(sqlcopysyntax)
var interndataset = [
['1', '4'],
['6', '12.074'],
['13.138', '16.183'],
['17.226', '21.605'],
['22.606', '24.733'],
['24.816', '27.027'],
['31.657', '33.617'],
['34.66', '37.204'],
['37.287', '38.58'],
['39.456', '43.669'],
['43.752', '47.297'],
['47.381', '49.55'],
];
var started = false;
var internmap = through2.obj(function(arr, enc, cb) {
var rowText = (started ? '\n' : '') + arr.join('\t');
started = true;
cb(null, rowText);
})
data.forEach(function(r) {
internmap.write(r);
})
internmap.end();
internmap.pipe(stream);

https://www.postgresql.org/docs/current/static/sql-copy.html
DELIMITER
Specifies the character that separates columns within each row (line)
of the file. The default is a tab character in text format, a comma in
CSV format. This must be a single one-byte character. This option is
not allowed when using binary format.
try using not default delimiter (as tabulation can be replaced on copy/paste), eg:
t=# create table intern(starttime float,endtime float);
CREATE TABLE
t=# \! cat 1
COPY intern(starttime,endtime) FROM STDIN delimiter ';';
1;4
6;12.074
13.138;16.183
17.226;21.605
22.606;24.733
24.816;27.027
31.657;33.617
34.66;37.204
37.287;38.58
39.456;43.669
43.752;47.297
47.381;49.55
49.633;54.68
54.763;58.225
59.142;62.98
64.189;68.861
69.82;71.613
72.364;76.201
76.285;78.787
78.871;81.832
\.
t=# \i 1
COPY 20
Also in your question you lack \., try typing in psql - you will see instructions:
t=# COPY intern(starttime,endtime) FROM STDIN delimiter ';';
Enter data to be copied followed by a newline.
End with a backslash and a period on a line by itself.
End with a backslash and a period on a line by itself.

Related

How to compare filenames with difference in special character encoding?

I am working with a system that syncs files between two vendors. The tooling is written in Javascript and does a transformation on file names before sending it to the destination. I am trying to fix a bug in it that is failing to properly compare file names between the origin and destination.
The script uses the file name to check if it's on destination
For example:
The following file name contains a special character that has different encoding between source and destination.
source: Chinchón.jpg // hex code: ó
destination : Chinchón.jpg // hex code: 0xf3
The function that does the transformation is:
export const normalizeText = (text:string) => text
.normalize('NFC')
.replace(/\p{Diacritic}/gu, "")
.replace(/\u{2019}/gu, "'")
.replace(/\u{ff1a}/gu, ":")
.trim()
and the comparison is happening just like the following:
const array1 = ['Chinchón.jpg'];
console.log(array1.includes('Chinchón.jpg')); // false
Do I reverse the transformation before comparing? what's the best way to do that?
If i got your question right:
// prepare dictionary
const rawDictionary = ['Chinchón.jpg']
const dictionary = rawDictionary.map(x => normalizeText(x))
...
const rawComparant = 'Chinchón.jpg'
const comparant = normalizeText(rawComparant)
console.log(rawSources.includes(comparant))

Validate row length when importing a txt file with fix length columns in SSIS 2016

I have a fixed length file that I have to read and validate. That file is produced by another system, but sometimes, employees are making manual changes to it. Example:
Layout
Variable: Surname size: 30 1 -30
Variable: Name size: 30 31-60
Variable: Email size: 30 61-90
Variable: Comments size: 30 91-120
Variable: CarriageReturn size: 2 121-123
So the system produces the following text file:
Source file
But then there is a manual intervention and the person does not respect the column length:
Source file after manual intervention
So before even starting to validate the values in the columns, everything is offset because my first carriage return is now splitting my "Comments" column when I read it in the SSIS.
Is there a way to tell the system that, if the length row is more than 2033, output in error file and continue ? What is the best way to do this?
Mylene
I found it!!
//Pass the file path and file name to the StreamReader and StreamWriter constructors
StreamReader sr = new StreamReader(inputFile);
StreamWriter sw = new StreamWriter(Dts.Connections["CE802CleanInput"].ConnectionString);
StreamWriter swe = new StreamWriter(Dts.Connections["CE802PreValidationErrors"].ConnectionString);
//Read the first line
line = sr.ReadLine();
while (line != null)
{
int length = line.Length;
if (length > 2033)
{
if
{
swe.WriteLine("Some records have been rejected at the pre validation phase.");
swe.WriteLine("Those records will not be included in the process.");
swe.WriteLine("Please review the records below, fix and re submit if applicable.");
swe.WriteLine("Input file: " + Dts.Connections["CE802Input"].ConnectionString.ToString());
swe.WriteLine();
swe.WriteLine(line);
count++;
}
else
{
swe.WriteLine(line);
count++;
}
}
if (length <= 2033)
{
sw.WriteLine(line);
}
line = sr.ReadLine();
}

Node-red output a message to CSV

For the past couple of days I have been trying and reading to get something very specific done in Node-Red: I want to send the (LoRa) message to a CSV.
This CSV should contain the following items:
topic
date
payload
I can insert the date using a function node:
var str1 = Date();
I have been playing around with CSV node, but I can't get it to output comma separated values. All this has probably to do with my lack of javascript programming skills, which is why I turn to you.
Can you help me out?
Edit:
I'm still looking for the answer, which has brought me the following:
Function node:
var res = Date() + "," + msg.topic + "," + msg.payload; return [ { payload: res } ];
Output:
[{"col1":"Mon Oct 17 2016 17:10:20 GMT+0200 (CEST)","col2":"1/test/1","col3":"string1"}]
All I want now is to lose the extra information such as column names and [{}]
The CSV node works only on the msg.payload field so you will have to copy the extra data into the payload object to get it to output what you want.
So to format the data correctly you need to place a function node with the following before the CSV node:
var originalPayload = msg.payload;
var newPayload = {};
newPayload.date = new Date().toString();
newPayload.topic = msg.topic;
newPayload.payload = originalPayload;
msg.payload = newPayload;
return msg;
And configure the CSV node to output columns "date,topic,payload"

Writing at end of every line in a file in Node.js?

I have a file
one
two
three
I want to append a word at the end of every line in this file. How can I achieve that in node ?
eg.
onecandy
twocandy
threecandy
Then I want to use this file in another function ,i.e after allcandy has been added . How do i do that ?
Because you will have to read the line to know where is ending and also you have to write at the end of the each line.
In conclusion you have to read everything and write at the end of each line just appending won't save to much performance it only complicate the things.
var fs = require("fs");
var allLines = fs.readFileSync('./input.txt').toString().split('\n');
fs.writeFileSync('./input.txt', '', function(){console.log('file is empty')})
allLines.forEach(function (line) {
var newLine = line + "candy";
console.log(newLine);
fs.appendFileSync("./input.txt", newLine.toString() + "\n");
});
// each line would have "candy" appended
allLines = fs.readFileSync('./input.txt').toString().split('\n');
Note: For replacing just some specified lines you can go through this answer.

NodeSchool IO Exercies 3

I've started learning node.js
I'm currently on exercise 3, where we have to, based on a file buffer, calculate the number of new line characters "\n"
I pass the tester but somehow if I create my own file file.txt, I am able to get the buffer, and print out the string, but it is unable to calculate the number of new lines (console.log(newLineNum)) returns 0
Here is the code
//import file system module
var fs = require("fs");
//get the buffer object based on argv[2]
var buf = fs.readFileSync(process.argv[2]);
//convert buffer to string
var str_buff = buf.toString();
//length of str_buff
var str_length = str_buff.length;
var numNewLines = 0;
for (var i = 0; i < str_length; i ++)
{
if(str_buff.charAt(i) == '\n')
{
numNewLines++;
}
}
console.log(numNewLines);
If i understand your question correctly, you are trying to get the line length of current file.
From the documentation:
The first element will be 'node', the second element will be the name
of the JavaScript file.
So you should replace process.argv[2] with process.argv[1].
Edit:
If you are passing a parameter for a file name on command-line like:
node server.py 'test.txt'
your code should work without any problem.
Your code is fine. You should check the file that you are using for the input.

Resources