AutoML prediction model PDF extraction: Required field not set - node.js

I'm trying to perform entity extraction on my prediction model but having issues passing PDF document from GCS.
/**
* TODO(developer): Uncomment these variables before running the sample.
*/
const projectId = "id";
const location = "us-central1";
const modelId = "modelid";
const keyFilename = "./gcp.json";
// Imports the Google Cloud AutoML library
const { PredictionServiceClient } = require("#google-cloud/automl").v1;
// Instantiates a client
const client = new PredictionServiceClient({ keyFilename });
async function predict() {
// Construct request
const request = {
name: client.modelPath(projectId, location, modelId),
payload: {
document: {
input_config: {
gcs_source: {
input_uris: "gs://pathtofile.pdf",
},
},
},
},
};
const [response] = await client.predict(request);
for (const annotationPayload of response.payload) {
console.log(annotationPayload);
console.log(`Text Extract Entity Types: ${annotationPayload.displayName}`);
console.log(`Text Score: ${annotationPayload.textExtraction.score}`);
const textSegment = annotationPayload.textExtraction.textSegment;
console.log(`Text Extract Entity Content: ${textSegment.content}`);
console.log(`Text Start Offset: ${textSegment.startOffset}`);
console.log(`Text End Offset: ${textSegment.endOffset}`);
}
}
predict();
This gives the following error:
Error: 3 INVALID_ARGUMENT: List of found errors: 1.Field: payload.document.document_text.content; Message: Required field not set.
code: 3,
details: 'List of found errors:\t1.Field: payload.document.document_text.content; Message: Required field not set.\t',
metadata: Metadata {
internalRepr: Map(3) {
'grpc-server-stats-bin' => [
Buffer(10) [Uint8Array] [
0, 0, 183, 198, 95,
1, 0, 0, 0, 0
]
],
'google.rpc.badrequest-bin' => [
Buffer(67) [Uint8Array] [
10, 65, 10, 38, 112, 97, 121, 108, 111, 97, 100,
46, 100, 111, 99, 117, 109, 101, 110, 116, 46, 100,
111, 99, 117, 109, 101, 110, 116, 95, 116, 101, 120,
116, 46, 99, 111, 110, 116, 101, 110, 116, 18, 23,
82, 101, 113, 117, 105, 114, 101, 100, 32, 102, 105,
101, 108, 100, 32, 110, 111, 116, 32, 115, 101, 116,
46
]
],
'grpc-status-details-bin' => [
Buffer(222) [Uint8Array] [
8, 3, 18, 104, 76, 105, 115, 116, 32, 111, 102, 32,
102, 111, 117, 110, 100, 32, 101, 114, 114, 111, 114, 115,
58, 9, 49, 46, 70, 105, 101, 108, 100, 58, 32, 112,
97, 121, 108, 111, 97, 100, 46, 100, 111, 99, 117, 109,
101, 110, 116, 46, 100, 111, 99, 117, 109, 101, 110, 116,
95, 116, 101, 120, 116, 46, 99, 111, 110, 116, 101, 110,
116, 59, 32, 77, 101, 115, 115, 97, 103, 101, 58, 32,
82, 101, 113, 117, 105, 114, 101, 100, 32, 102, 105, 101,
108, 100, 32, 110,
... 122 more items
]
]
},
options: {}
},
statusDetails: [
BadRequest {
fieldViolations: [
FieldViolation {
field: 'payload.document.document_text.content',
description: 'Required field not set.'
}
]
}
]
}
I have checked all the passed parameters and they're fine.
I'm using this as an example: https://cloud.google.com/natural-language/automl/docs/predict#automl_language_entity_extraction_predict-nodejs
It works fine when I pass text but I would like to pass PDF as the payload.
Thank you

Related

Process stdout capture for Autoruns

Hey guys I am working on a project currently where I am trying to run Autorunsc64.exe from the sysinternals suite and, unlike other executables I have tried, I can't get it to give me the output in a form I want. Here is the current iteration of my code:
use std::process::{Command as process_command, Stdio};
fn autorun_programs() -> String {
// Check where sysinternals is developer vs release
let full_exe_path = current_exe().unwrap();
let mut split_exe_path: Vec<&str> = Vec::new();
if full_exe_path.to_str().unwrap().contains("target") {
let temp: Vec<&str> = full_exe_path.to_str().unwrap().split("system_recon\\target\\debug\\system_recon.exe").collect();
split_exe_path.push(temp[0]);
} else {
let temp: Vec<&str> = full_exe_path.to_str().unwrap().split("system_recon.exe").collect();
split_exe_path.push(temp[0]);
};
let partial_exe_path = split_exe_path[0].to_string();
let sysinternals_exe_string = partial_exe_path + &"SysinternalsSuite\\Autorunsc64.exe".to_string();
//my_command.args(["-nobanner", "/accepteula", "-a *", "-c", "-h", "-s", "-v", "-vt", "*"]);
let mut command = process_command::new(sysinternals_exe_string);
command.arg("-nobanner");
command.arg("-accepteula");
//command.arg("-x");
command.arg("-t");
//command.arg("-a");
//command.arg("*");
//command.arg("-x");
//command.arg("-h");
//command.arg("-s");
//command.arg("-v");
//command.arg("-vt");
//command.arg("*");
command.stdout(Stdio::piped());
command.stderr(Stdio::piped());
let output = command.execute_output().unwrap();
if let Some(exit_code) = output.status.code() {
if exit_code == 0 {
println!("Ok.");
} else {
eprintln!("Failed.");
}
} else {
eprintln!("Interrupted!");
}
println!("{}", String::from_utf8(output.stdout).unwrap());
return "Bruh".to_string()
}
The code outputs this:
thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: FromUtf8Error { bytes: [255, 254, 13, 0, 10, 0, 72, 0, 75, 0, 76, 0, 77, 0, 92, 0, 83, 0, 121, 0, 115, 0, 116, 0, 101, 0, 109, 0, 92, 0, 67, 0, 117, 0, 114, 0, 114, 0, 101, 0, 110, 0, 116, 0, 67, 0, 111, 0, 110, 0, 116, 0, 114, 0, 111, 0, 108, 0, 83, 0, 101, 0, 116, 0, 92, 0, 67, 0, 111, 0, 110, 0, 116, 0, 114, 0, 111, 0, 108, 0, 92, 0, 84, 0, 101, 0, 114, 0, 109, 0, 105, 0, 110, 0, 97, 0, 108, 0, 32, 0, 83, 0, 101, 0, 114, 0, 118, 0, 101, 0, 114, 0, 92, 0, 87, 0, 100, 0, 115, 0, 92, 0, 114, 0, 100, 0, 112, 0, 119, 0, 100, 0, 92, 0, 83, 0, 116, 0, 97, 0, 114, 0, 116, 0, 117, 0, 112, 0, 80, 0, 114, 0, 111, 0, 103, 0, 114, 0, 97, 0, 109, 0, 115, 0, 13, 0, 10, 0, 32, 0, 32, 0, 32, 0, 114, 0, 100, 0, 112, 0, 99, 0, 108, 0, 105, 0, 112, 0, 13, 0, 10, 0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 114, 0, 100, 0, 112, 0, 99, 0, 108, 0, 105, 0, 112, 0, 13, 0, 10, 0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 82, 0, 68, 0, 80, 0, 32, 0, 67, 0, 108, 0, 105, 0, 112, 0, 98, 0, 111, 0, 97, 0, 114, 0, 100, 0, 32, 0, 77, 0, 111, 0, 110, 0, 105, 0, 116, 0, 111, 0, 114, 0, 13, 0, 10, 0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 77, 0, 105, 0, 99, 0, 114, 0, 111, 0, 115, 0, 111, 0, 102, 0, 116, 0, 32, 0, 67, 0, 111, 0, 114, 0, 112, 0, 111, 0, 114, 0, 97, 0, 116, 0, 105, 0, 111, 0, 110, 0, 13, 0, 10, 0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 49, 0, 48, 0, 46, 0, 48, 0, 46, 0, 49, 0, 57, 0, 48, 0, 52, 0, 49, 0, 46, 0, 55, 0, 52, 0, 54, 0, 13, 0, 10, 0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 99, 0, 58, 0, 92, 0, 119, 0, 105, 0, 110, 0, 100, 0, 111, 0, 119, 0, 115, 0, 92, 0, 115, 0, 121, 0, 115, 0, 116, 0, 101, 0, 109, 0, 51, 0, 50, 0, 92, 0, 114, 0, 100, 0, 112, 0, 99, 0, 108, 0, 105, 0, 112, 0, 46, 0, 101, 0, 120, 0, 101, 0, 13, 0, 10, 0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 50, 0, 48, 0, 48, 0, 55, 0, 48, 0, 49, 0, 50, 0, 54, 0, 45, 0, 48, 0, 50, 0, 48, 0, 48, 0, 51, 0, 56, 0, 13, 0, 10, 0, 13, 0, 10, 0, 72, 0, 75, 0, 76, 0, 77, 0, 92, 0, 83, 0, 79, 0, 70, 0, 84, 0, 87, 0, 65, 0, 82, 0, 69, 0, 92, 0, 77, 0, 105, 0, 99, 0, 114, 0, 111, 0, 115, 0, 111, 0, 102, 0, 116, 0, 92, 0, 87, 0, 105, 0, 110, 0, 100, 0, 111, 0, 119, 0, 115, 0, 32, 0, 78, 0, 84, 0, 92, 0, 67, 0, 117, 0, 114, 0, 114, 0, 101, 0, 110, 0, 116, 0, 86, 0, 101, 0, 114, 0, 115, 0, 105, 0, 111, 0, 110, 0, 92, 0, 87, 0, 105, 0, 110, 0, 108, 0, 111, 0, 103, 0, 111, 0, 110, 0, 92, 0, 85, 0, 115, 0, 101, 0, 114, 0, 105, 0, 110, 0, 105, 0, 116, 0, 13, 0, 10, 0, 32, 0, 32, 0, 32, 0, 67, 0, 58, 0, 92, 0, 87, 0, 105, 0, 110, 0, 100, 0, 111, 0, 119, 0, 115, 0, 92, 0, 115, 0, 121, 0, 115, 0, 116, 0, 101, 0, 109, 0, 51, 0, 50, 0, 92, 0, 117, 0, 115, 0, 101, 0, 114, 0, 105, 0, 110, 0, 105, 0
That is just a sample from the output. Seems like it is not valid utf8 or something idk I am really stumped on this one. I also tried with create_process_w but that had its own problems. Any help is appreciated, thanks!
Turns out, this is UTF-16. So you must first convert it from bytes into u16s:
let u16s: Vec<u16> = output.stdout.chunks_exact(2).map(|chunk| u16::from_ne_bytes([chunk[0], chunk[1]])).collect();
let s = String::from_utf16(&u16s).unwrap(); // may want to use from_utf16_lossy?
Also note that the output starts with a byte order mark: "\u{feff}\r\nHKLM". You may want to strip it off if it's present.
Ok so this is my final solution for my problem:
fn parse_utf16_bytes(bytes: &[u8]) -> Option<String> {
let mut chunks = bytes.chunks_exact(2);
let is_big_endian = match chunks.next() {
Some(&[254, 255]) => true,
Some(&[255, 254]) => false,
_ => return None,
};
let utf16: Vec<_> = chunks
.map(|x| {
let arr2 = x.try_into().expect("convert .chunks_exact() to [u8; 2]");
if is_big_endian {
u16::from_be_bytes(arr2)
} else {
u16::from_le_bytes(arr2)
}
})
.collect();
String::from_utf16(&utf16).ok()
}
Thanks for everybody that helped :)

Node.js child_process issuewhile running in VS code

Below is the ERROR I am getting when I run following 2 lines of code
RUNNING IN VS CODE
ONLY 2 FOLLOWING LINES cause this trouble pls help me get started!!!!!!!!!
const cp = require("child_process");
cp.execSync("calc");
status: 127,
signal: null,
output: [
null,
Buffer(0) [Uint8Array] [],
Buffer(33) [Uint8Array] [
47, 98, 105, 110, 47, 115, 104, 58,
32, 99, 97, 108, 99, 58, 32, 99,
111, 109, 109, 97, 110, 100, 32, 110,
111, 116, 32, 102, 111, 117, 110, 100,
10
]
],
pid: 19417,
stdout: Buffer(0) [Uint8Array] [],
stderr: Buffer(33) [Uint8Array] [
47, 98, 105, 110, 47, 115, 104, 58,
32, 99, 97, 108, 99, 58, 32, 99,
111, 109, 109, 97, 110, 100, 32, 110,
111, 116, 32, 102, 111, 117, 110, 100,
10
]
}
Check if calc is installed and can be found. 127 indicates that the program is not found.

Can we use EJS tags with ChartsJS?

<script>
var ctx = document.getElementById('myChart').getContext('2d');
var myChart = new Chart(ctx, {
type: 'pie',
data: {
datasets: [{
labels: <%- JSON.stringify(state_names); %>
label: '# of Votes',
data: <%- JSON.stringify(confirm_cases); %>,
backgroundColor: [
'rgba(255, 99, 132, 0.2)',
'rgba(54, 162, 235, 0.2)',
'rgba(255, 206, 86, 0.2)',
'rgba(75, 192, 192, 0.2)',
'rgba(153, 102, 255, 0.2)',
'rgba(255, 159, 64, 0.2)'
],
borderColor: [
'rgba(255, 99, 132, 1)',
'rgba(54, 162, 235, 1)',
'rgba(255, 206, 86, 1)',
'rgba(75, 192, 192, 1)',
'rgba(153, 102, 255, 1)',
'rgba(255, 159, 64, 1)'
],
borderWidth: 1
}]
}
});
</script>
Above is my code to plot a chart using Charts.js. However, The labels array gives me a problem. The data field of charts is successfully incorporated. Can someone please help me out with this? "state_names" and "confirm_cases" are arrays being sent from the app.js file in the node project
Your labels array is in the wrong space, it has to be in the data object on the same level as the datasets object is like this:
<script>
var ctx = document.getElementById('myChart').getContext('2d');
var myChart = new Chart(ctx, {
type: 'pie',
data: {
labels: < % -JSON.stringify(state_names); % >
datasets: [{
label: '# of Votes',
data: < % -JSON.stringify(confirm_cases); % > ,
backgroundColor: [
'rgba(255, 99, 132, 0.2)',
'rgba(54, 162, 235, 0.2)',
'rgba(255, 206, 86, 0.2)',
'rgba(75, 192, 192, 0.2)',
'rgba(153, 102, 255, 0.2)',
'rgba(255, 159, 64, 0.2)'
],
borderColor: [
'rgba(255, 99, 132, 1)',
'rgba(54, 162, 235, 1)',
'rgba(255, 206, 86, 1)',
'rgba(75, 192, 192, 1)',
'rgba(153, 102, 255, 1)',
'rgba(255, 159, 64, 1)'
],
borderWidth: 1
}]
}
});
</script>

ng2-chartjs2 ignores type and always shows a bar chart

I am using angular2 with ng2-chartjs2. I have the following component and template which I took directly from https://www.npmjs.com/package/ng2-chartjs2. The chart displays fine, but when I change the type in the template from bar to line I still see the same bar chart with no errors.
import { Component } from '#angular/core';
import { Router, RouterLink, CanActivate } from '#angular/router';
import { CORE_DIRECTIVES } from '#angular/common';
import { DashboardLayoutComponent } from '../dashboard_layout/dashboard_layout.component';
import {ChartComponent, Chart} from 'ng2-chartjs2';
#Component({
selector: 'home',
templateUrl: 'client/components/home/home.component.html',
directives: [DashboardLayoutComponent, CORE_DIRECTIVES, ChartComponent]
})
export class HomeComponent {
constructor(private _router: Router) {
}
labels: string[] = ["Red", "Blue", "Yellow", "Green", "Purple", "Orange"];
data: Chart.Dataset[] = [
{
label: '# of Votes',
data: [12, 19, 3, 5, 25, 3],
backgroundColor: [
'rgba(255, 99, 132, 0.2)',
'rgba(54, 162, 235, 0.2)',
'rgba(255, 206, 86, 0.2)',
'rgba(75, 192, 192, 0.2)',
'rgba(153, 102, 255, 0.2)',
'rgba(255, 159, 64, 0.2)'
],
borderColor: [
'rgba(255,99,132,1)',
'rgba(54, 162, 235, 1)',
'rgba(255, 206, 86, 1)',
'rgba(75, 192, 192, 1)',
'rgba(153, 102, 255, 1)',
'rgba(255, 159, 64, 1)'
],
borderWidth: 1
}
];
}
Template:
<!-- template -->
<dashboard-layout pageTitle="Statistical Overview">
<div class="home">
<chart [labels]="labels" [data]="data" type="line"></chart>
</div>
</dashboard-layout>
It seems like I am following the documentation correctly. Is this a bug? Is there a workaround if so?
Look at the source code
if(!this.options){
this.options = {
type: 'bar', <== this line
data: {
labels: this.labels,
datasets: this.data
}
}
}
This way if you don't provide options it always will be as bar chart
You can leverage the following workaround:
import { ChartComponent } from 'ng2-chartjs2';
#Component({
selector: 'my-app',
template: `
<chart [options]="options"></chart>
`,
directives: [ChartComponent]
})
export class AppComponent {
options = {
type: 'line',
data: {
labels: ["Red", "Blue", "Yellow", "Green", "Purple", "Orange"],
datasets: [
{
label: '# of Votes',
data: [12, 19, 3, 5, 25, 3],
backgroundColor: [
'rgba(255, 99, 132, 0.2)',
'rgba(54, 162, 235, 0.2)',
'rgba(255, 206, 86, 0.2)',
'rgba(75, 192, 192, 0.2)',
'rgba(153, 102, 255, 0.2)',
'rgba(255, 159, 64, 0.2)'
],
borderColor: [
'rgba(255,99,132,1)',
'rgba(54, 162, 235, 1)',
'rgba(255, 206, 86, 1)',
'rgba(75, 192, 192, 1)',
'rgba(153, 102, 255, 1)',
'rgba(255, 159, 64, 1)'
],
borderWidth: 1
}
]
}
};
}
Demo Plunker

inserts not working mongodb

I'm having a problem while using a webtask as a backend web service to a static website. I'm using the web service to insert values to a mongodb database. The inserts are not returning any errors, but there is nothing inserting into the database. It just says 'worked' on the response. Here's the relevant code:
var MongoClient = require('mongodb').MongoClient;
function savePage (page, db, cb){
var doc = {
pageSec: page
};
db.collection("page").insertOne(doc, function (err) {
if(err) {return cb(err);}
cb('worked');
});
}
module.exports = function(context, cb){
var section = context.query.section;
if(section){
MongoClient.connect(context.data.mongoUri, function(err, db) {
if(err) {return cb(err);}
savePage(section, db, function (err) {
if(err){return cb(err);}
cb(null);
});
});
}
};
additionally, this is what mongodb returns in response to the insert
"details": {
"ok": 1,
"n": 1,
"lastOp": "6307485423539060737",
"electionId": "576dec4e2c52240b7a5bca5e"
},
"message": {
"parsed": true,
"index": 96,
"raw": {
"type": "Buffer",
"data": [
96,
0,
0,
0,
163,
69,
36,
3,
5,
0,
0,
0,
1,
0,
0,
0,
8,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
60,
0,
0,
0,
16,
111,
107,
0,
1,
0,
0,
0,
16,
110,
0,
1,
0,
0,
0,
17,
108,
97,
115,
116,
79,
112,
0,
1,
0,
0,
0,
147,
176,
136,
87,
7,
101,
108,
101,
99,
116,
105,
111,
110,
73,
100,
0,
87,
109,
236,
78,
44,
82,
36,
11,
122,
91,
202,
94,
0
]
},
"data": {
"type": "Buffer",
"data": [
96,
0,
0,
0,
163,
69,
36,
3,
5,
0,
0,
0,
1,
0,
0,
0,
8,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
60,
0,
0,
0,
16,
111,
107,
0,
1,
0,
0,
0,
16,
110,
0,
1,
0,
0,
0,
17,
108,
97,
115,
116,
79,
112,
0,
1,
0,
0,
0,
147,
176,
136,
87,
7,
101,
108,
101,
99,
116,
105,
111,
110,
73,
100,
0,
87,
109,
236,
78,
44,
82,
36,
11,
122,
91,
202,
94,
0
]
},
"bson": {},
"opts": {
"promoteLongs": true
},
"length": 96,
"requestId": 52708771,
"responseTo": 5,
"responseFlags": 8,
"cursorId": "0",
"startingFrom": 0,
"numberReturned": 1,
"documents": [
{
"ok": 1,
"n": 1,
"lastOp": "6307485423539060737",
"electionId": "576dec4e2c52240b7a5bca5e"
}
],
"cursorNotFound": false,
"queryFailure": false,
"shardConfigStale": false,
"awaitCapable": true,
"promoteLongs": true
}
When I build my JSON object to sent to MongoDB, I always set the property name as string and it works.
var doc = {
"pageSec": page
};
I don't see anything wrong in your code, so you can try this.

Resources