Python Elasticsearch 7.05 indexing failing - python-3.x

I have created an index on my ES server via Kibana that looks like this
PUT export_control
{
"mappings": {
"properties": {
"content": {
"type": "text",
"term_vector": "with_positions_offsets"
}
}
}
}
Result is
{
"acknowledged" : true,
"shards_acknowledged" : true,
"index" : "export_control"
}
When I try to index a new document with the following command in python
col_names = df.columns
for row_number in range(df.shape[0]):
body = dict([(name, str(df.iloc[row_number][name])) for name in col_names])
es.index(index = 'export_control', doc_type="pdf_document", body = body)
I get the following error
RequestError Traceback (most recent call last)
<ipython-input-247-6df868fd60f1> in <module>
2 for row_number in range(df.shape[0]):
3 body = dict([(name, str(df.iloc[row_number][name])) for name in col_names])
----> 4 es.index(index = 'export_control', doc_type="pdf_document", body = body)
/usr/local/lib/python3.7/site-packages/elasticsearch/client/utils.py in _wrapped(*args, **kwargs)
74 for p in es_query_params + GLOBAL_PARAMS:
75 if p in kwargs:
---> 76 v = kwargs.pop(p)
77 if v is not None:
78 params[p] = _escape(v)
/usr/local/lib/python3.7/site-packages/elasticsearch/client/__init__.py in index(self, index, doc_type, body, id, params)
317 "timeout",
318 "timestamp",
--> 319 "ttl",
320 "version",
321 "version_type",
/usr/local/lib/python3.7/site-packages/elasticsearch/transport.py in perform_request(self, method, url, headers, params, body)
316 method = "POST"
317
--> 318 # or as source parameter
319 elif self.send_get_body_as == "source":
320 if params is None:
/usr/local/lib/python3.7/site-packages/elasticsearch/connection/http_urllib3.py in perform_request(self, method, url, params, body, timeout, ignore, headers)
183 kw.update(
184 {
--> 185 "cert_reqs": "CERT_REQUIRED",
186 "ca_certs": ca_certs,
187 "cert_file": client_cert,
/usr/local/lib/python3.7/site-packages/elasticsearch/connection/base.py in _raise_error(self, status_code, raw_data)
123 )
124 logger.debug("> %s", body)
--> 125 logger.debug("< %s", response)
126
127 self._log_trace(method, path, body, status_code, response, duration)
RequestError: TransportError(400, 'illegal_argument_exception', 'Rejecting mapping update to [export_control] as the final mapping would have more than 1 type: [_doc, pdf_document]')

The problem is that in Elasticsearch types were removed recently - some full definition of it, which means, that you could have only one type - called _doc, while you're creating second one - pdf_document.
You should use
es.index(index = 'export_control', doc_type="_doc", body = body)

Related

I want to know how to read the value of headers in SetTask class after configuring tasks = [SetTask]?

from locust import HttpUser, task, between,SequentialTaskSet,TaskSet,User,events
import json
#events.test_start.add_listener
def on_start(**kwargs):
print("A test will start...")
#events.test_stop.add_listener
def on_stop(**kwargs):
print("A test is ending...")
class SetTask(TaskSet):
#task
def getLogDetail(self):
deatil_url = "/auth/online?page=0&size=10&sort=id%2Cdesc"
with self.client.request(method='GET',
url=deatil_url,
headers=self.headers,
name='get log detail') as response:
print(response.text)
class FlaskTask(SequentialTaskSet):
def __init__(self,*args,**kwargs):
super().__init__(*args,**kwargs)
def on_start(self):
res = self.client.request(method='GET', url="/auth/code",name="get code")
uuid = res.json()['uuid']
headers = {
"content-type": "application/json;charset=UTF-8",
}
datda_info = {
"code": "8",
"password": "B0GdcVWB+XtTwsyBjzoRkn8VnSgtPVKpl8mp7AuQ+BTeU030grUkRwmOHXFCjEhKXB7yezBS7dFEJ63ykR2piQ==",
"username": "admin",
"uuid": uuid
}
with self.client.request(method='POST',url="/auth/login", headers=headers, catch_response=True,data=json.dumps(datda_info),name="get token") as response:
self.token = response.json()['token']
if response.status_code == 200:
self.token = response.json()['token']
response.success()
else:
response.failure("get token failed")
self.headers = {
"Authorization": self.token
}
tasks = [SetTask]
#task
def getUserDetail(self):
deatil_url = "/api/dictDetail?dictName=user_status&page=0&size=9999"
with self.client.request(method='GET',
url=deatil_url,
headers=self.headers,
name='get user detail') as response:
print(response.text)
def function_task():
print("This is the function task")
class FlaskUser(HttpUser):
host = 'http://192.168.31.243:8888'
wait_time = between(1,3)
tasks = [FlaskTask,function_task]
I got error:
A test will start...
[2022-03-26 00:04:56,146] wangguilin/INFO/locust.runners: Ramping to 1 users at a rate of 1.00 per second
[2022-03-26 00:04:56,147] wangguilin/INFO/locust.runners: All users spawned: {"FlaskUser": 1} (1 total users)
[2022-03-26 00:04:56,148] wangguilin/ERROR/locust.user.task: function_task() takes 0 positional arguments but 1 was given
Traceback (most recent call last):
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 319, in run
self.execute_next_task()
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 344, in execute_next_task
self.execute_task(self._task_queue.pop(0))
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 457, in execute_task
task(self.user)
TypeError: function_task() takes 0 positional arguments but 1 was given
[2022-03-26 00:04:58,154] wangguilin/ERROR/locust.user.task: 'SetTask' object has no attribute 'headers'
Traceback (most recent call last):
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 319, in run
self.execute_next_task()
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 344, in execute_next_task
self.execute_task(self._task_queue.pop(0))
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 356, in execute_task
task(self)
File "C:\Users\Users\PycharmProjects\testlocust\locustflask.py", line 22, in getLogDetail
headers=self.headers,
AttributeError: 'SetTask' object has no attribute 'headers'
Traceback (most recent call last):
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\gevent_ffi\loop.py", line 270, in python_check_callback
def python_check_callback(self, watcher_ptr): # pylint:disable=unused-argument
question:
I want to know how to read the value of headers in SetTask class after configuring tasks = [SetTask]?
my locust version is: 2.8.3
So can parameters be passed in this case?

Server returning 404 error while consuming functions of BSE SOAP Apis

I was integrating bsestar mf soap api's with django. So in order to consume soap apis i have use python package called zeep. but after creating its client on its end point and after that whenever i try to run soap functions it says 404 error.
my code :
import zeep
def soap_set_wsa_headers(method_url, svc_url):
header = zeep.xsd.Element(
"Header",
zeep.xsd.ComplexType(
[
zeep.xsd.Element(
"{http://www.w3.org/2005/08/addressing}Action", zeep.xsd.String()
),
zeep.xsd.Element(
"{http://www.w3.org/2005/08/addressing}To", zeep.xsd.String()
),
]
),
)
header_value = header(Action=method_url, To=svc_url)
return header_value
WSDL_ORDER_URL = [
'https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc?singleWsdl',
'https://www.bsestarmf.in/MFOrderEntry/MFOrder.svc?singleWsdl'
]
SVC_ORDER_URL = [
'https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc',
'https://www.bsestarmf.in/MFOrderEntry/MFOrder.svc'
]
METHOD_ORDER_URL = [
'https://bsestarmfdemo.bseindia.com/MFOrderEntry/',
'https://bsestarmf.in/MFOrderEntry/'
]
WSDL_UPLOAD_URL = [
'https://bsestarmfdemo.bseindia.com/MFUploadService/MFUploadService.svc?singleWsdl',
'https://www.bsestarmf.in/StarMFWebService/StarMFWebService.svc?singleWsdl'
]
SVC_UPLOAD_URL = [
'https://bsestarmfdemo.bseindia.com/MFUploadService/MFUploadService.svc/Basic',
'https://www.bsestarmf.in/StarMFWebService/StarMFWebService.svc/Basic'
]
METHOD_UPLOAD_URL = [
'https://bsestarmfdemo.bseindia.com/2016/01/IMFUploadService/',
'https://www.bsestarmf.in/2016/01/IStarMFWebService/'
]
client = zeep.Client(wsdl=WSDL_ORDER_URL[0])
method_url = METHOD_UPLOAD_URL[0] + 'getPassword'
svc_url = SVC_UPLOAD_URL[0]
header_value=soap_set_wsa_headers(method_url,svc_url)
client.service.getPassword(UserId='3081801',
Password='Abc4231',
PassKey='anything',
_soapheaders=[header_value])
Error while calling getPassword function:
zeep.transports: HTTP Post to
https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc:
<?xml version='1.0' encoding='utf-8'?>
<soap-env:Envelope xmlns:soap-env="http://www.w3.org/2003/05/soap-envelope"><soap-env:Header xmlns:wsa="http://www.w3.org/2005/08/addressing"><Header><wsa:Action>https://bsestarmfdemo.bseindia.com/2016/01/IMFUploadService/getPassword</wsa:Action><wsa:To>https://bsestarmfdemo.bseindia.com/MFUploadService/MFUploadService.svc/Basic</wsa:To></Header><wsa:Action>http://bsestarmf.in/MFOrderEntry/getPassword</wsa:Action><wsa:MessageID>urn:uuid:953b29c6-a2a0-48a0-aef0-5d4f1547b39a</wsa:MessageID><wsa:To>https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc</wsa:To></soap-env:Header><soap-env:Body><ns0:getPassword xmlns:ns0="http://bsestarmf.in/"><ns0:UserId>3081801</ns0:UserId><ns0:Password>Abc4231</ns0:Password><ns0:PassKey>kuchbhi</ns0:PassKey></ns0:getPassword></soap-env:Body></soap-env:Envelope>
zeep.transports: HTTP Response from https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc (status: 404):
---------------------------------------------------------------------------
TransportError Traceback (most recent call last)
<ipython-input-64-b5411107d6e5> in <module>
----> 1 client.service.getPassword(UserId='3081801',
2 Password='Abc4231',
3 PassKey='kuchbhi',
4 _soapheaders=[header_value])
/opt/anaconda3/lib/python3.8/site-packages/zeep/proxy.py in __call__(self, *args, **kwargs)
44 kwargs["_soapheaders"] = soap_headers
45
---> 46 return self._proxy._binding.send(
47 self._proxy._client,
48 self._proxy._binding_options,
/opt/anaconda3/lib/python3.8/site-packages/zeep/wsdl/bindings/soap.py in send(self, client, options, operation, args, kwargs)
133 return response
134
--> 135 return self.process_reply(client, operation_obj, response)
136
137 async def send_async(self, client, options, operation, args, kwargs):
/opt/anaconda3/lib/python3.8/site-packages/zeep/wsdl/bindings/soap.py in process_reply(self, client, operation, response)
179
180 elif response.status_code != 200 and not response.content:
--> 181 raise TransportError(
182 u"Server returned HTTP status %d (no content available)"
183 % response.status_code,
TransportError: Server returned HTTP status 404 (no content available)
Thanks in advance :)

Specifying ConnectionProperties When Creating New Connection in Boto3

I'm trying to create a connection using a boto3 python script. It's a connection to a mysql database on an ec2 instance. I'm using the script below. I'm getting the error message below. I'm not sure what I'm missing. I've used similar scripts to create glue crawlers without issue. I'm pretty much following the boto3 documentation except I added username and password in the ConnectionProperties. I'm not sure if that's correct. Can someone please let me know if I'm doing this right, or else what I need to fix in my code?
code:
# create new connection
response = client.create_connection(
ConnectionInput={
'Name': 'tst_scrpt',
'ConnectionType': 'JDBC',
'ConnectionProperties': {
'string': 'jdbc:mysql://dataxxx:3306/disxxx',
'username':'root',
'password':'ip1k5PNCxxxxx'
}
,
'PhysicalConnectionRequirements': {
'SubnetId': 'subnet-0436167b7cbxxxx',
'SecurityGroupIdList': [
'sg-02c3f45ce51exxxxx'
]
}
}
)
Error:
---------------------------------------------------------------------------
InvalidInputException Traceback (most recent call last)
<ipython-input-18-6ac0bdcfa816> in <module>
16 'SubnetId': 'subnet-0436167b7cbxxxx',
17 'SecurityGroupIdList': [
---> 18 'sg-02c3f45ce51exxxxx'
19 ]
20 # ,
/anaconda3/envs/py36/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
355 "%s() only accepts keyword arguments." % py_operation_name)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358
359 _api_call.__name__ = str(py_operation_name)
/anaconda3/envs/py36/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
659 error_code = parsed_response.get("Error", {}).get("Code")
660 error_class = self.exceptions.from_code(error_code)
--> 661 raise error_class(parsed_response, operation_name)
662 else:
663 return parsed_response
InvalidInputException: An error occurred (InvalidInputException) when calling the CreateConnection operation: Validation for connection properties failed
I had the ConnectionProperties wrong. replace 'string' with 'JDBC_CONNECTION_URL'. The correction is below.
'ConnectionProperties': {
'JDBC_CONNECTION_URL': 'jdbc:mysql://dataxxx:3306/disxxx',
'username':'root',
'password':'ip1k5PNCxxxxx'
}

Flask TypeError: argument of type 'NoneType' is not iterable

I am not sure why I am getting this TypeError:
File "C:/Users/PycharmProjects/REST/app.py", line 30, in
valid_book_object
if ("isbn" in book and "name" in book and "price" in book):
TypeError: argument of type 'NoneType' is not iterable
127.0.0.1 - - [12/Nov/2018 14:22:29] "POST /books HTTP/1.1" 500 -
Code:
from flask import Flask, jsonify, request
from test import *
app=Flask(__name__)
books=[
{'name': 'M',
'price': 6.75,
'isbn':123
},
{'name': 'G',
'price': 7.75,
'isbn':456
},
]
#GET /store
#app.route('/books') #first endpoint
def get_books():
return jsonify({'books': books})
# POST /books
#{'name': 'F',
#'price': 7.00,
#'isbn': 789
#},
def valid_book_object(book):
if ("isbn" in book and "name" in book and "price" in book):
return True
print("true")
else:
return False
print("false")
#app.route('/books', methods=['POST'])
def add_book():
#return jsonify(request.get_json())
request_data=request.get_json()
if(valid_book_object(request_data)):
books.insert(0, request_data)
return "True"
else:
return "False"
#GET /books/456
#app.route('/books/<int:isbn>') #second endpoint
def get_book_by_isbn(isbn):
return_value={}
for book in books:
if book["isbn"]==isbn:
return_value={
'name': book["name"],
'price':book["price"]
}
return jsonify(return_value)
app.run(port=5000)
You are not sending JSON data to /books route using POST method.
I tried your solution with postman and it worked. Also, if you want to use some route for GET and POST, don't split them. Use methods=['GET', 'POST']. Here is your code reformatted with PEP 8 standard:
from flask import Flask, jsonify, request
app = Flask(__name__)
books = [
{'name': 'M',
'price': 6.75,
'isbn': 123
},
{'name': 'G',
'price': 7.75,
'isbn': 456
}
]
# POST /books
# {
# "name": "F",
# "price": 7.00,
# "isbn": 789
# }
def valid_book_object(book):
if "isbn" in book and "name" in book and "price" in book:
return True
else:
return False
#app.route('/books', methods=['GET', 'POST'])
def add_book():
# If request is GET, just return JSON data of books.
if request.method == 'GET':
return jsonify({'books': books})
else:
# This is part if it is POST request
request_data = request.get_json()
if valid_book_object(request_data):
books.insert(0, request_data)
return "True"
else:
return "False"
# GET /books/456
#app.route('/books/<int:isbn>') # second endpoint
def get_book_by_isbn(isbn):
return_value = {}
for book in books:
if book["isbn"] == isbn:
return_value = {
'name': book["name"],
'price': book["price"]
}
return jsonify(return_value)
return 'No book with {} isbn value'.format(isbn)
if __name__ == '__main__':
app.run(port=5000)
And here is the output from postman (you can see True at the bottom, that is what you return if POST was successful):
If you use postman, be sure to select application/json content-type.
If you are using JQuery ajax method, please read this answer. But anyway, here is using JQuery (tested):
data = JSON.stringify({
name: "F",
price: 7.00,
isbn: 789
});
$.ajax({
url: '/books',
type: "POST",
data: data,
contentType: "application/json",
dataType: "json",
success: function(){
console.log('Post executed successfully');
}
})

Error in Watson Classifier API reference nodejs example: source.on is not a function

I'm trying to use Watson Classifier from node. I've started by implementing the example in the API reference, found at https://www.ibm.com/watson/developercloud/natural-language-classifier/api/v1/node.html?node#create-classifier
My code (sensitive information replaced with stars):
58 create: function(args, cb) {
59 var params = {
60 metadata: {
61 language: 'en',
62 name: '*********************'
63 },
64 training_data: fs.createReadStream(config.data.prepared.training)
65 };
66
67 params.training_data.on("readable", function () {
68 nlc.createClassifier(params, function(err, response) {
69 if (err)
70 return cb(err);
71 console.log(JSON.stringify(response, null, 2));
72 cb();
73 });
74 });
75 },
The file I am trying to make a stream from exists. The stream works (I've managed to read from it on "readable"). I've placed the on("readable") part because it made sense for me to do all of this once the stream becomes available, and also because I wanted to be able to check that I can read from it. It does not change the outcome, however.
nlc is the natural_langauge_classifier instance.
I'm getting this:
octav#****************:~/watsonnlu$ node nlc.js create
/home/octav/watsonnlu/node_modules/delayed-stream/lib/delayed_stream.js:33
source.on('error', function() {});
^
TypeError: source.on is not a function
at Function.DelayedStream.create (/home/octav/watsonnlu/node_modules/delayed-stream/lib/delayed_stream.js:33:10)
at FormData.CombinedStream.append (/home/octav/watsonnlu/node_modules/combined-stream/lib/combined_stream.js:44:37)
at FormData.append (/home/octav/watsonnlu/node_modules/form-data/lib/form_data.js:74:3)
at appendFormValue (/home/octav/watsonnlu/node_modules/request/request.js:321:21)
at Request.init (/home/octav/watsonnlu/node_modules/request/request.js:334:11)
at new Request (/home/octav/watsonnlu/node_modules/request/request.js:128:8)
at request (/home/octav/watsonnlu/node_modules/request/index.js:53:10)
at Object.createRequest (/home/octav/watsonnlu/node_modules/watson-developer-cloud/lib/requestwrapper.js:208:12)
at NaturalLanguageClassifierV1.createClassifier (/home/octav/watsonnlu/node_modules/watson-developer-cloud/natural-language-classifier/v1-generated.js:143:33)
at ReadStream.<anonymous> (/home/octav/watsonnlu/nlc.js:68:8)
I tried debugging it myself for a while, but I'm not sure what this source is actually supposed to be. It's just an object composed of the metadata I put in and the "emit" function if I print it before the offending line in delayed-stream.js.
{ language: 'en',
name: '*******************',
emit: [Function] }
This is my package.json file:
1 {
2 "name": "watsonnlu",
3 "version": "0.0.1",
4 "dependencies": {
5 "csv-parse": "2.0.0",
6 "watson-developer-cloud": "3.2.1"
7 }
8 }
Any ideas how to make the example work?
Cheers!
Octav
I got the answer in the meantime thanks to the good people at IBM. It seems you have to send the metadata as a stringified JSON:
59 var params = {
60 metadata: JSON.stringify({
61 language: 'en',
62 name: '*********************'
63 }),
64 training_data: fs.createReadStream(config.data.prepared.training)
65 };

Resources