I have a problem on logstash where my workers jump to 100% CPU while activating 1 specific filter. Im currently using this input + filter (on which grok parses ok):
input {
redis {
key => "logs_services_dyna_customer.lnxdev01"
data_type => ['list']
db => 8
host => "kibana.test.local"
type => "D_Customer"
tags => ["d_local"]
add_field => {
server => "lnxdev01"
}
codec => "plain"
}
}
filter {
if [type] in ["D_Customer"] {
mutate {
gsub => ["message", "\r", ""]
}
grok {
patterns_dir => "/etc/logstash/patterns/"
match => ["message", "^Request Headers:\n(%{WORD:request_method} %{DATA:service_url} %{DATA}\n)?(Host: %{HOSTNAME:request_host}\n)?(Connection: %{WORD:request_connection}\n)?(User-Agent: %{DATA:user_agent}\n)?(Content-Type: %{DATA:content_type}\n)?(SOAPAction: \"%{DATA:request_action}\"\n)?(Content-Length: %{NUMBER:request_content_length}\n)?(Cookie: %{DATA}\n)?(:?X-START-DATE: %{APACHE_DATETIME:request_start_date})?\n+Request Body:\n%{GREEDIERDATA:request_body}\n+Response Headers:\n(HTTP/%{DATA}(?<response_code>\d{3}) %{DATA:response_message}\n)?(Cache-Control: %{DATA}\n)?(Content-Length: %{NUMBER:response_content_length}\n)?(Content-Type: %{DATA:response_content_type}\n)?(Server: %{DATA:response_server}\n)?(X-AspNet-Version: %{DATA}\n)?(X-Powered-By: %{DATA}\n)?(Date: %{APACHE_DATETIME:response_date}\n)?(Connection: %{WORD:response_connection}\n)?\n+Response Body:\n%{GREEDIERDATA:response_body}$"]
add_field => ["category", "Service"]
}
date {
match => ["request_start_date", "EEE, dd MMMM YYYY HH:mm:ss z"]
}
date {
match => ["request_start_date", "EEE, dd MMMM YYYY HH:mm:ss z"]
target => "request_start_date"
}
date {
match => ["response_date", "EEE, dd MMMM YYYY HH:mm:ss z"]
target => "response_date"
}
}
}
And below the log itself:
Request Headers:
POST /Middleware.OIL.I01_SearchCustomer.Wcf/WcfService_Middleware_OIL_I01_SearchCustomer_App.svc HTTP/1.1
Host: w12vfdbiz101-t.dy.nl
Connection: close
User-Agent: PHP-SOAP/7.0.9-1+deb.sury.org~trusty+1
Content-Type: text/xml; charset=utf-8
SOAPAction: "SearchCustomer"
Content-Length: 1628
X-START-DATE: Mon, 07 Nov 2016 13:27:03 UTC
Request Body:
<?xml version="1.0" encoding="UTF-8"?>
<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns1="http://www.dy.eu/Customer/V1-0"><SOAP-ENV:Body><v1:SearchCustomerRequest xmlns:v1="http://www.dy.eu/Customer/V1-0" xmlns:v2="http://dy/CoreComponents/Schemas/dy/V2-0" xmlns:urn="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" xmlns:urn1="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" xmlns:v21="http://dy/CoreComponents/Schemas/dy/V2-0">
<v2:HeaderInfo>
<v2:TransactionId>58208127a1179</v2:TransactionId>
<v2:UserId>1</v2:UserId>
<v2:ApplicationId>Omnius</v2:ApplicationId>
<v2:PartyName>OGW</v2:PartyName>
</v2:HeaderInfo>
<v1:SearchedAccount>
<urn:PartyName>
<urn1:Name/>
</urn:PartyName>
</v1:SearchedAccount>
<v1:SearchedContact>
<v21:Role>
<v21:Person>
<urn1:ID/>
<urn1:FirstName>Friedrich</urn1:FirstName>
<urn1:FamilyName>Muller</urn1:FamilyName>
<urn:Contact>
<urn1:Telephone/>
<urn1:ElectronicMail/>
</urn:Contact>
<urn:ResidenceAddress>
<urn1:StreetName/>
<urn1:BuildingNumber/>
<urn1:CityName/>
<urn1:PostalZone/>
</urn:ResidenceAddress>
</v21:Person>
</v21:Role>
</v1:SearchedContact>
<v1:SearchedEquipment>
<urn:ManufacturersItemIdentification>
<urn1:ID/>
</urn:ManufacturersItemIdentification>
</v1:SearchedEquipment>
</v1:SearchCustomerRequest></SOAP-ENV:Body></SOAP-ENV:Envelope>
Response Headers:
HTTP/1.1 200 OK
Cache-Control: private
Content-Length: 2379
Content-Type: text/xml; charset=utf-8
Server: Microsoft-IIS/8.5
X-AspNet-Version: 4.0.30319
X-Powered-By: ASP.NET
Date: Mon, 07 Nov 2016 13:27:10 GMT
Connection: close
Response Body:
<s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/"><s:Body><ns0:SearchCustomerResponse xmlns:ns0="http://www.dy.eu/Customer/V1-0" xmlns:ns1="http://uri.etsi.org/01903/v1.3.2#" xmlns:ns2="urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2" xmlns:ns3="urn:oasis:names:specification:ubl:schema:xsd:SignatureBasicComponents-2" xmlns:ns4="http://www.w3.org/2000/09/xmldsig#" xmlns:ns5="urn:oasis:names:specification:ubl:schema:xsd:QualifiedDataTypes-2" xmlns:ns6="urn:oasis:names:specification:ubl:schema:xsd:UnqualifiedDataTypes-2" xmlns:ns7="urn:un:unece:uncefact:data:specification:CoreComponentTypeSchemaModule:2" xmlns:ns8="urn:oasis:names:specification:ubl:schema:xsd:SignatureAggregateComponents-2" xmlns:ns9="http://uri.etsi.org/01903/v1.4.1#" xmlns:ns10="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" xmlns:ns11="http://Dy/CoreComponents/Schemas/Dy/V2-0" xmlns:ns12="urn:oasis:names:specification:ubl:schema:xsd:CommonSignatureComponents-2" xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><cbc:ReferenceID>8</cbc:ReferenceID><ns10:Status xsi:type="ns10:StatusType"><cbc:StatusReasonCode/><cbc:StatusReason/></ns10:Status><ns11:Success>true</ns11:Success><ns0:ReturnedNumberOfRecords>1</ns0:ReturnedNumberOfRecords><ns11:Account><ns10:PartyName><cbc:Name>CompanyName_0</cbc:Name></ns10:PartyName><ns10:PostalAddress xsi:type="ns10:AddressType"><cbc:StreetName>Street0</cbc:StreetName><cbc:BuildingNumber>1</cbc:BuildingNumber><cbc:CityName>City0</cbc:CityName><cbc:PostalZone>12345</cbc:PostalZone></ns10:PostalAddress><ns10:Contact xsi:type="ns10:ContactType"><cbc:ID>KS_12366</cbc:ID><cbc:ElectronicMail/></ns10:Contact><ns11:AccountCategory>KD</ns11:AccountCategory><ns11:AccountClass>P</ns11:AccountClass><ns11:ParentAccountNumber><cbc:ID>555</cbc:ID></ns11:ParentAccountNumber><ns11:BankAccountNumber>1234</ns11:BankAccountNumber><ns11:Role><ns11:Person><cbc:FirstName>Friedrich</cbc:FirstName><cbc:FamilyName>Müller</cbc:FamilyName><cbc:BirthDate>19700101</cbc:BirthDate><ns11:ContactPhoneNumber><ns11:PhoneNumber>0612387777</ns11:PhoneNumber><ns11:Type>Mobile</ns11:Type></ns11:ContactPhoneNumber></ns11:Person></ns11:Role></ns11:Account></ns0:SearchCustomerResponse></s:Body></s:Envelope>
So the question is, how come that logstash is spiking to 100% CPU and is this because of below filter? Should it be different or can it be more efficient?
Thanks
Pim
The issue was that somebody used GREEDIERDATA. This meant it will match on .* \n \r. If grok needs to parse it will get in a massive loop since it will not know when to stop at all and needs to iterate the complete entry all the time
Related
On the command line, I can do a request like: curl -I -H "Fastly-Debug: 1"
and it will return a lot of helpful information from the CDN serving that URL, in this case, Fastly:
cache-control: public, max-age=0, must-revalidate
last-modified: Tue, 20 Apr 2021 21:17:46 GMT
etag: "4c5cb3eb0ddb001584dad329b8727a9a"
content-type: text/html
server: AmazonS3
surrogate-key: /v3.6/tutorial/nav/alerts-and-monitoring/
accept-ranges: bytes
date: Fri, 30 Apr 2021 20:50:15 GMT
via: 1.1 varnish
age: 0
fastly-debug-path: (D cache-lga21923-LGA 1619815815) (F cache-lga21940-LGA 1619815815)
fastly-debug-ttl: (M cache-lga21923-LGA - - 0)
fastly-debug-digest: 04c3e527819b6a877de6577f7461e132b97665100a63ca8f667d87d049092233
x-served-by: cache-lga21923-LGA
x-cache: MISS
x-cache-hits: 0
x-timer: S1619815815.944515,VS0,VE136
vary: Accept-Encoding
content-length: 65489
How can I do this in node?
This is my attempt:
const headers = {
'Fastly-Key': environment.getFastlyToken(),
'Accept': 'application/json',
'Content-Type': 'application/json',
'Fastly-Debug': 1
};
async retrieveSurrogateKey(url) {
console.log("this is the url: ", url)
try {
request({
method: `GET`,
url: url,
headers: headers,
}, function(err, response, body) {
if (err){
console.trace(err)
}
console.log(request.headers)
})
} catch (error) {
console.log("error in retrieval: ", error)
}
}
Is there a way for me to pass in the -I and -H flags?
The -H (header) flag allows you to specify a custom header in cURL. Your code already does this - great! All that's left is emulating the -I (head) flag.
From cURL manpage for the -I option:
Fetch the headers only! HTTP-servers feature the command HEAD which this uses to get nothing but the header of a document.
To use the HEAD method, you need to specify it instead of GET:
method: `HEAD`
And finally, the headers returned by the server in the response can be obtained from response.headers.
I discovered that axios is returning a string instead of a valid json.
headers:Object {date: "Tue, 02 Jun 2020 08:44:06 GMT", server: "Apache", connection: "close", …}
connection:"close"
content-type:"text/html; charset=UTF-8"
date:"Tue, 02 Jun 2020 08:44:06 GMT"
server:"Apache"
transfer-encoding:"chunked"
how do I change the content-type to application/json in NestJs application?
I tried this but didnt not work
const meterInfo = await this.httpService.get(url, { headers: { "Content-Type": "application/json" } }).toPromise();
Here is the invalid json returned.
"{"status":"00","message":"OK","access_token":"2347682423567","customer":{"name":"John Doe","address":"Mr. John Doe 34 Tokai, leaflet. 7999.","util":"Demo Utility","minimumAmount":"13897"},"response_hash":"c43c9d74480f340f55156f6r5c56487v8w"}"
Instead of sending a Content-Type header, you should send an Accept header with the same MIME type. This tells the server what you are expecting to receive, and if Content Negotiation is set up properly, it will allow you to get a JSON back instead of that weird string.
this.httpService.get(
url,
{
headers: {
'Accept': 'application/json',
},
},
).toPromise();
If that doesn't work, you'll need to provide your own serializer to take the string from that wonky format to JSON, or get in touch with the server admins and see if they can provide you better documentation about how to consume their API.
I created a server with actix_web that will connect through GET to another service using actix client and return body on success or error on error. I have been able to return the body but have no clue about how to return the error.
This is my handler:
fn echo_client(client: web::Data<Client>) -> impl Future<Item = HttpResponse, Error = Error> {
client
.get("127.0.0.1:9596/echo/javier") // <- Create request builder
.header("User-Agent", "Actix-web")
//.finish().unwrap()
.send() // <- Send http request
.map_err(|_| ())
//.map_err(Error::from)
.and_then(|response| {
response
.body()
.and_then(|body| {
println!("{:?}", body);
Ok(HttpResponse::Ok().body(body))
})
.map_err(|error| Err(error.error_response()))
})
}
There are three things that may fail:
Failing connection.
Non 200-status code.
Abrupt stop in body stream.
To handle 1, do not map_err to ():
.map_err(|err| match err {
SendRequestError::Connect(error) => {
ErrorBadGateway(format!("Unable to connect to httpbin: {}", error))
}
error => ErrorInternalServerError(error),
})
SendRequestError lists the errors that can occur when doing client requests.
To handle 2, make sure you use the status code from the client response:
.and_then(|response| Ok(HttpResponse::build(response.status()).streaming(response))))
actix-web handles 3 I believe.
Complete example, handling headers too:
use actix_web::client::{Client, SendRequestError};
use actix_web::error::{ErrorBadGateway, ErrorInternalServerError};
use actix_web::{web, App, Error, HttpResponse, HttpServer};
use futures::future::Future;
fn main() {
HttpServer::new(|| App::new().data(Client::new()).route("/", web::to(handler)))
.bind("127.0.0.1:8000")
.expect("Cannot bind to port 8000")
.run()
.expect("Unable to run server");
}
fn handler(client: web::Data<Client>) -> Box<Future<Item = HttpResponse, Error = Error>> {
Box::new(
client
.get("https://httpbin.org/get")
.no_decompress()
.send()
.map_err(|err| match err {
SendRequestError::Connect(error) => {
ErrorBadGateway(format!("Unable to connect to httpbin: {}", error))
}
error => ErrorInternalServerError(error),
})
.and_then(|response| {
let mut result = HttpResponse::build(response.status());
let headers = response
.headers()
.iter()
.filter(|(h, _)| *h != "connection" && *h != "content-length");
for (header_name, header_value) in headers {
result.header(header_name.clone(), header_value.clone());
}
Ok(result.streaming(response))
}),
)
}
Which actually failed:
$ curl -v localhost:8000
* Rebuilt URL to: localhost:8000/
* Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to localhost (127.0.0.1) port 8000 (#0)
> GET / HTTP/1.1
> Host: localhost:8000
> User-Agent: curl/7.54.0
> Accept: */*
>
< HTTP/1.1 502 Bad Gateway
< content-length: 50
< content-type: text/plain
< date: Sun, 07 Jul 2019 21:01:39 GMT
<
* Connection #0 to host localhost left intact
Unable to connect to httpbin: SSL is not supported
Add ssl as a feature in Cargo.toml to fix the connection error:
actix-web = { version = "1.0", features=["ssl"] }
Then try the request again:
$ curl -v localhost:8000
* Rebuilt URL to: localhost:8000/
* Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to localhost (127.0.0.1) port 8000 (#0)
> GET / HTTP/1.1
> Host: localhost:8000
> User-Agent: curl/7.54.0
> Accept: */*
>
< HTTP/1.1 200 OK
< transfer-encoding: chunked
< x-frame-options: DENY
< date: Sun, 07 Jul 2019 21:07:18 GMT
< content-type: application/json
< access-control-allow-origin: *
< access-control-allow-credentials: true
< server: nginx
< x-content-type-options: nosniff
< x-xss-protection: 1; mode=block
< referrer-policy: no-referrer-when-downgrade
<
{
"args": {},
"headers": {
"Date": "Sun, 07 Jul 2019 21:07:18 GMT",
"Host": "httpbin.org"
},
"origin": "212.251.175.90, 212.251.175.90",
"url": "https://httpbin.org/get"
}
Been troubleshooting for a while and can't come up with an answer. I run get commands with no problem so list basecamps, todos, accounts, etc. All that's working fine. It's when I POST to todolists.json where I'm having problems.
I'm sending to this URL:
https://3.basecampapi.com/3266090/buckets/1940710/todosets/296431359/todolists.json
Headers:
[0] => Content-Type: application/json; charset=utf-8
[1] => User-Agent: Template Deployer (s****d#***.org)
[2] => Authorization: Bearer [working token]
[3] => If-None-Match:
POST payload:
name=foobar
And here's the response I'm getting:
[result] => Array
(
[status] => 400
[error] => Bad Request
)
[headers] => Array
(
[Server] => nginx
[Date] => Fri, 18 Nov 2016 21:05:44 GMT
[Content-Type] => application/json; charset=utf-8
[Content-Length] => 36
[Connection] => keep-alive
[X-Request-Id] => ab71cfaa-e24c-4a38-b1af-44fe8ae8d147
[X-Runtime] => 0.456205
[Strict-Transport-Security] => max-age=15552000; includeSubDomains; preload
[X-Queue-Time] => 0.0010805130004882812
[Set-Cookie] => force-primary-dc=true;Max-Age=5;Path=/;Version=1;
)
[code] => 400
[content_type] => application/json; charset=utf-8
I've tried several combinations of name, description and project (from another thread I found about someone else's problem), to no avail. I don't even know what else to try.
Any Ideas?
And it's because I'm not a smart person :)
Answer: the payload needs to be json encoded: {"content":"Do this!"} rather than url encoded: name=foobar
Problem Solved!
I’d like to ask you a favor. I do have a problem confirming previously readed D2C message using IoT Hub. I am using REST API to pick up message like (I have replace SIG)
Request:
GET https://iot-hub-pospa.azure-devices.net/devices/18596c88-01e6-3f16-427b-10028d7305c5/messages/devicebound?api-version=2015-08-15-preview HTTP/1.1
IoTHub-MessageLockTimeout: 3600
Accept: application/json
Authorization: SharedAccessSignature sr=iot-hub-pospa.azure-devices.net&sig={sig}&se=1485558838&skn=iothubowner
Host: iot-hub-pospa.azure-devices.net
If-None-Match: "1c5006a4-2288-4a2f-b7ea-dcdf9b5bbc99"
Connection: Close
X-P2P-PeerDist: Version=1.1
X-P2P-PeerDistEx: MinContentInformation=1.0, MaxContentInformation=2.0
Accept-Encoding: peerdist
Response:
HTTP/1.1 200 OK
Content-Length: 35
ETag: "dfc78580-d251-4156-a5f6-c2a30811a504"
Server: Microsoft-HTTPAPI/2.0
iothub-messageid: 02cdb012-9749-48a9-bfb3-5812a4740675
iothub-to: /devices/18596c88-01e6-3f16-427b-10028d7305c5/messages/deviceBound
iothub-expiry:
iothub-correlationid:
iothub-ack: full
iothub-sequencenumber: 56
iothub-enqueuedtime: 2/2/2016 9:57:34 AM
iothub-deliverycount: 0
Date: Tue, 02 Feb 2016 10:21:43 GMT
Connection: close
2/2/2016 10:57:34 AM - Test message
Then when confirming I am getting HTTP 412 like
Request:
DELETE https://iot-hub-pospa.azure-devices.net/devices/18596c88-01e6-3f16-427b-10028d7305c5/messages/devicebound/02cdb012-9749-48a9-bfb3-5812a4740675?api-version=2015-08-15-preview HTTP/1.1
Accept: application/json
If-Match: "02cdb012-9749-48a9-bfb3-5812a4740675"
Authorization: SharedAccessSignature sr=iot-hub-pospa.azure-devices.net&sig={sig}&se=1485558838&skn=iothubowner
Host: iot-hub-pospa.azure-devices.net
Content-Length: 0
Connection: Close
Response:
HTTP/1.1 412 Precondition Failed
Content-Length: 330
Content-Type: application/json; charset=utf-8
Server: Microsoft-HTTPAPI/2.0
iothub-errorcode: DeviceMessageLockLost
Date: Tue, 02 Feb 2016 10:21:49 GMT
Connection: close
{"Message":"ErrorCode:DeviceMessageLockLost;Message 02cdb012-9749-48a9-bfb3-5812a4740675 lock was lost for Device 18596c88-01e6-3f16-427b-10028d7305c5\r\nTracking Id:05994074a3664933a0910b5fc70e04e5-G:GatewayWorkerRole.6-B:1-P:cffe397b-f627-4435-bd54-48f5ba79c3ca-TimeStamp:02/02/2016 10:21:49\r\nErrorCode:DeviceMessageLockLost"}
Does anybody know what should I do to successfully confirm/delete message from IoT Hub, please? Thanks
static DeviceClient _deviceClient;
_deviceClient = DeviceClient.CreateFromConnectionString(<IoTHubURI>, TransportType.Http1);
public void SendMessage(IDictionary<string, object> dictionary)
{
Microsoft.Azure.Devices.Client.Message message = new Microsoft.Azure.Devices.Client.Message();
try
{
foreach (var r in dictionary)
{
message.Properties[r.Key] = r.Value.ToString();
}
_deviceClient.SendEventAsync(message);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
}