Specifying ConnectionProperties When Creating New Connection in Boto3 - python-3.x

I'm trying to create a connection using a boto3 python script. It's a connection to a mysql database on an ec2 instance. I'm using the script below. I'm getting the error message below. I'm not sure what I'm missing. I've used similar scripts to create glue crawlers without issue. I'm pretty much following the boto3 documentation except I added username and password in the ConnectionProperties. I'm not sure if that's correct. Can someone please let me know if I'm doing this right, or else what I need to fix in my code?
code:
# create new connection
response = client.create_connection(
ConnectionInput={
'Name': 'tst_scrpt',
'ConnectionType': 'JDBC',
'ConnectionProperties': {
'string': 'jdbc:mysql://dataxxx:3306/disxxx',
'username':'root',
'password':'ip1k5PNCxxxxx'
}
,
'PhysicalConnectionRequirements': {
'SubnetId': 'subnet-0436167b7cbxxxx',
'SecurityGroupIdList': [
'sg-02c3f45ce51exxxxx'
]
}
}
)
Error:
---------------------------------------------------------------------------
InvalidInputException Traceback (most recent call last)
<ipython-input-18-6ac0bdcfa816> in <module>
16 'SubnetId': 'subnet-0436167b7cbxxxx',
17 'SecurityGroupIdList': [
---> 18 'sg-02c3f45ce51exxxxx'
19 ]
20 # ,
/anaconda3/envs/py36/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
355 "%s() only accepts keyword arguments." % py_operation_name)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358
359 _api_call.__name__ = str(py_operation_name)
/anaconda3/envs/py36/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
659 error_code = parsed_response.get("Error", {}).get("Code")
660 error_class = self.exceptions.from_code(error_code)
--> 661 raise error_class(parsed_response, operation_name)
662 else:
663 return parsed_response
InvalidInputException: An error occurred (InvalidInputException) when calling the CreateConnection operation: Validation for connection properties failed

I had the ConnectionProperties wrong. replace 'string' with 'JDBC_CONNECTION_URL'. The correction is below.
'ConnectionProperties': {
'JDBC_CONNECTION_URL': 'jdbc:mysql://dataxxx:3306/disxxx',
'username':'root',
'password':'ip1k5PNCxxxxx'
}

Related

GEE python:“ee.batch.Export.image.toAsset” "Request had insufficient authentication scopes."

I am getting an error when using python GEE api to save processed image in Colab. The code and the error are as follow:
# Load a landsat image and select three bands.
landsat = ee.Image('LANDSAT/LC08/C01/T1_TOA/LC08_123032_20140515')\
.select(['B4', 'B3', 'B2'])
# Create a geometry representing an export region.
geometry = ee.Geometry.Rectangle([116.2621, 39.8412, 116.4849, 40.01236])
center = geometry.centroid().getInfo()['coordinates']
center.reverse()
Mapdisplay(center,{'landsat':landsat.getMapId()},zoom_start=7)
# Get band 4 from the Landsat image, copy it.
band4 = landsat.select('B4').rename('b4_mean')\
.addBands(landsat.select('B4').rename('b4_sample'))\
.addBands(landsat.select('B4').rename('b4_max'))\
# Export the image to an Earth Engine asset.
task = ee.batch.Export.image.toAsset(**{
'image': band4,
'description': 'imageToAssetExample',
'assetId': 'users/csaybar/exampleExport',
'scale': 100,
'region': geometry.getInfo()['coordinates']
})
task.start()
This error occurs due to "task.start()"
WARNING:googleapiclient.http:Invalid JSON content from response: b'{\n "error": {\n "code": 403,\n "message": "Request had insufficient authentication scopes.",\n "status": "PERMISSION_DENIED",\n "details": [\n {\n "#type": "type.googleapis.com/google.rpc.ErrorInfo",\n "reason": "ACCESS_TOKEN_SCOPE_INSUFFICIENT",\n "domain": "googleapis.com",\n "metadata": {\n "method": "google.earthengine.v1alpha.EarthEngine.ExportImage",\n "service": "earthengine.googleapis.com"\n }\n }\n ]\n }\n}\n'
---------------------------------------------------------------------------
HttpError Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/ee/data.py in _execute_cloud_call(call, num_retries)
327 try:
--> 328 return call.execute(num_retries=num_retries)
329 except googleapiclient.errors.HttpError as e:
6 frames
HttpError: <HttpError 403 when requesting https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/image:export?alt=json returned "Request had insufficient authentication scopes.". Details: "[{'#type': 'type.googleapis.com/google.rpc.ErrorInfo', 'reason': 'ACCESS_TOKEN_SCOPE_INSUFFICIENT', 'domain': 'googleapis.com', 'metadata': {'method': 'google.earthengine.v1alpha.EarthEngine.ExportImage', 'service': 'earthengine.googleapis.com'}}]">
During handling of the above exception, another exception occurred:
EEException Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/ee/data.py in _execute_cloud_call(call, num_retries)
328 return call.execute(num_retries=num_retries)
329 except googleapiclient.errors.HttpError as e:
--> 330 raise _translate_cloud_exception(e)
331
332
EEException: Request had insufficient authentication scopes.
I tried to look for a GEE service in the google cloud console to enable it, but couldn't find one. And I want to konw how to fix this problem.
The most recent version of the google earth engine API seems to have introduced some issues with gcloud. Try rolling back to an earlier version using conda install -c conda-forge earthengine-api=0.1.320 and then rerunning your process.
For future reference, most GEE-related questions get asked and answered over at gis.stackexchange.com.

boto3 s3 bucket tagging

I'm getting access error while tagging a bucket. Please note that the role I'm using has s3 full access.
The code works fine till this point-
for bucket in s3.buckets.all():
s3_bucket = bucket
s3_bucket_name = s3_bucket.name
try:
response = s3_client.get_bucket_tagging(Bucket=s3_bucket_name)
print(response)
except ClientError:
print (s3_bucket_name, "does not have tags")
but after adding putTag code, it gives error even for GetBucketTagging operation.
This is my final code:
for bucket in s3.buckets.all():
s3_bucket = bucket
s3_bucket_name = s3_bucket.name
try:
response = s3_client.get_bucket_tagging(Bucket=s3_bucket_name)
print(response)
except ClientError:
print (s3_bucket_name, "does not have tags")
bucket_tagging = s3.BucketTagging(s3_bucket_name)
response = bucket_tagging.put(
Tagging={
'TagSet': [
{
'Key': 'pcs:name',
'Value': s3_bucket_name
},
]
},
)
The error I'm getting is-
botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the GetBucketTagging operation: Access Denied
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "tagging.py", line 91, in <module>
tagging()
File "tagging.py", line 71, in tagging
'Value': s3_bucket_name
File "/home/ec2-user/compass_backend/compass_backend/lib64/python3.7/site-packages/boto3/resources/factory.py", line 520, in do_action
response = action(self, *args, **kwargs)
File "/home/ec2-user/compass_backend/compass_backend/lib64/python3.7/site-packages/boto3/resources/action.py", line 83, in __call__
response = getattr(parent.meta.client, operation_name)(*args, **params)
File "/home/ec2-user/compass_backend/compass_backend/lib64/python3.7/site-packages/botocore/client.py", line 395, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/ec2-user/compass_backend/compass_backend/lib64/python3.7/site-packages/botocore/client.py", line 725, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the PutBucketTagging operation: Access Denied
am I passing the tag parameters wrong? Got this from Boto3 documentation itself
I couldn't find a way to catch the exception, however, this worked for me:
tagging_client = boto3.client('resourcegroupstaggingapi')
s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
for bucket in s3.buckets.all():
s3_bucket = bucket
s3_bucket_name = s3_bucket.name
bucket_tagging = s3.BucketTagging(s3_bucket_name)
try:
response = s3_client.get_bucket_tagging(Bucket=s3_bucket_name)
a = response
except ClientError:
response = tagging_client.tag_resources(
ResourceARNList=[
"arn:aws:s3:::" + bucket.name
],
Tags={
'pcs:name': bucket.name
}
)
pls note that you'll need the additional "resource tagging" policy attached to your role.
Hope this helps. Cheers.
I took out the try sections and ran this version of your code:
import boto3
s3_resource = boto3.resource('s3')
bucket_tagging = s3_resource.BucketTagging('my-bucket-name')
response = bucket_tagging.put(
Tagging={
'TagSet': [
{
'Key': 'pcs:name',
'Value': 'stackoverflow'
},
]
},
)
It worked fine:
[
Therefore, there must be something else that is causing your request to fail. You might want to check AWS CloudTrail to see if there is a hint as to why the request was denied.
"get_bucket_tagging" throws NoSuchTagSet when there are no tags. for testing create a tag first before run test or Catch the exception and create tags.

I want to know how to read the value of headers in SetTask class after configuring tasks = [SetTask]?

from locust import HttpUser, task, between,SequentialTaskSet,TaskSet,User,events
import json
#events.test_start.add_listener
def on_start(**kwargs):
print("A test will start...")
#events.test_stop.add_listener
def on_stop(**kwargs):
print("A test is ending...")
class SetTask(TaskSet):
#task
def getLogDetail(self):
deatil_url = "/auth/online?page=0&size=10&sort=id%2Cdesc"
with self.client.request(method='GET',
url=deatil_url,
headers=self.headers,
name='get log detail') as response:
print(response.text)
class FlaskTask(SequentialTaskSet):
def __init__(self,*args,**kwargs):
super().__init__(*args,**kwargs)
def on_start(self):
res = self.client.request(method='GET', url="/auth/code",name="get code")
uuid = res.json()['uuid']
headers = {
"content-type": "application/json;charset=UTF-8",
}
datda_info = {
"code": "8",
"password": "B0GdcVWB+XtTwsyBjzoRkn8VnSgtPVKpl8mp7AuQ+BTeU030grUkRwmOHXFCjEhKXB7yezBS7dFEJ63ykR2piQ==",
"username": "admin",
"uuid": uuid
}
with self.client.request(method='POST',url="/auth/login", headers=headers, catch_response=True,data=json.dumps(datda_info),name="get token") as response:
self.token = response.json()['token']
if response.status_code == 200:
self.token = response.json()['token']
response.success()
else:
response.failure("get token failed")
self.headers = {
"Authorization": self.token
}
tasks = [SetTask]
#task
def getUserDetail(self):
deatil_url = "/api/dictDetail?dictName=user_status&page=0&size=9999"
with self.client.request(method='GET',
url=deatil_url,
headers=self.headers,
name='get user detail') as response:
print(response.text)
def function_task():
print("This is the function task")
class FlaskUser(HttpUser):
host = 'http://192.168.31.243:8888'
wait_time = between(1,3)
tasks = [FlaskTask,function_task]
I got error:
A test will start...
[2022-03-26 00:04:56,146] wangguilin/INFO/locust.runners: Ramping to 1 users at a rate of 1.00 per second
[2022-03-26 00:04:56,147] wangguilin/INFO/locust.runners: All users spawned: {"FlaskUser": 1} (1 total users)
[2022-03-26 00:04:56,148] wangguilin/ERROR/locust.user.task: function_task() takes 0 positional arguments but 1 was given
Traceback (most recent call last):
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 319, in run
self.execute_next_task()
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 344, in execute_next_task
self.execute_task(self._task_queue.pop(0))
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 457, in execute_task
task(self.user)
TypeError: function_task() takes 0 positional arguments but 1 was given
[2022-03-26 00:04:58,154] wangguilin/ERROR/locust.user.task: 'SetTask' object has no attribute 'headers'
Traceback (most recent call last):
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 319, in run
self.execute_next_task()
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 344, in execute_next_task
self.execute_task(self._task_queue.pop(0))
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\locust\user\task.py", line 356, in execute_task
task(self)
File "C:\Users\Users\PycharmProjects\testlocust\locustflask.py", line 22, in getLogDetail
headers=self.headers,
AttributeError: 'SetTask' object has no attribute 'headers'
Traceback (most recent call last):
File "c:\users\Users\pycharmprojects\testlocust\venv\lib\site-packages\gevent_ffi\loop.py", line 270, in python_check_callback
def python_check_callback(self, watcher_ptr): # pylint:disable=unused-argument
question:
I want to know how to read the value of headers in SetTask class after configuring tasks = [SetTask]?
my locust version is: 2.8.3
So can parameters be passed in this case?

Server returning 404 error while consuming functions of BSE SOAP Apis

I was integrating bsestar mf soap api's with django. So in order to consume soap apis i have use python package called zeep. but after creating its client on its end point and after that whenever i try to run soap functions it says 404 error.
my code :
import zeep
def soap_set_wsa_headers(method_url, svc_url):
header = zeep.xsd.Element(
"Header",
zeep.xsd.ComplexType(
[
zeep.xsd.Element(
"{http://www.w3.org/2005/08/addressing}Action", zeep.xsd.String()
),
zeep.xsd.Element(
"{http://www.w3.org/2005/08/addressing}To", zeep.xsd.String()
),
]
),
)
header_value = header(Action=method_url, To=svc_url)
return header_value
WSDL_ORDER_URL = [
'https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc?singleWsdl',
'https://www.bsestarmf.in/MFOrderEntry/MFOrder.svc?singleWsdl'
]
SVC_ORDER_URL = [
'https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc',
'https://www.bsestarmf.in/MFOrderEntry/MFOrder.svc'
]
METHOD_ORDER_URL = [
'https://bsestarmfdemo.bseindia.com/MFOrderEntry/',
'https://bsestarmf.in/MFOrderEntry/'
]
WSDL_UPLOAD_URL = [
'https://bsestarmfdemo.bseindia.com/MFUploadService/MFUploadService.svc?singleWsdl',
'https://www.bsestarmf.in/StarMFWebService/StarMFWebService.svc?singleWsdl'
]
SVC_UPLOAD_URL = [
'https://bsestarmfdemo.bseindia.com/MFUploadService/MFUploadService.svc/Basic',
'https://www.bsestarmf.in/StarMFWebService/StarMFWebService.svc/Basic'
]
METHOD_UPLOAD_URL = [
'https://bsestarmfdemo.bseindia.com/2016/01/IMFUploadService/',
'https://www.bsestarmf.in/2016/01/IStarMFWebService/'
]
client = zeep.Client(wsdl=WSDL_ORDER_URL[0])
method_url = METHOD_UPLOAD_URL[0] + 'getPassword'
svc_url = SVC_UPLOAD_URL[0]
header_value=soap_set_wsa_headers(method_url,svc_url)
client.service.getPassword(UserId='3081801',
Password='Abc4231',
PassKey='anything',
_soapheaders=[header_value])
Error while calling getPassword function:
zeep.transports: HTTP Post to
https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc:
<?xml version='1.0' encoding='utf-8'?>
<soap-env:Envelope xmlns:soap-env="http://www.w3.org/2003/05/soap-envelope"><soap-env:Header xmlns:wsa="http://www.w3.org/2005/08/addressing"><Header><wsa:Action>https://bsestarmfdemo.bseindia.com/2016/01/IMFUploadService/getPassword</wsa:Action><wsa:To>https://bsestarmfdemo.bseindia.com/MFUploadService/MFUploadService.svc/Basic</wsa:To></Header><wsa:Action>http://bsestarmf.in/MFOrderEntry/getPassword</wsa:Action><wsa:MessageID>urn:uuid:953b29c6-a2a0-48a0-aef0-5d4f1547b39a</wsa:MessageID><wsa:To>https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc</wsa:To></soap-env:Header><soap-env:Body><ns0:getPassword xmlns:ns0="http://bsestarmf.in/"><ns0:UserId>3081801</ns0:UserId><ns0:Password>Abc4231</ns0:Password><ns0:PassKey>kuchbhi</ns0:PassKey></ns0:getPassword></soap-env:Body></soap-env:Envelope>
zeep.transports: HTTP Response from https://bsestarmfdemo.bseindia.com/MFOrderEntry/MFOrder.svc (status: 404):
---------------------------------------------------------------------------
TransportError Traceback (most recent call last)
<ipython-input-64-b5411107d6e5> in <module>
----> 1 client.service.getPassword(UserId='3081801',
2 Password='Abc4231',
3 PassKey='kuchbhi',
4 _soapheaders=[header_value])
/opt/anaconda3/lib/python3.8/site-packages/zeep/proxy.py in __call__(self, *args, **kwargs)
44 kwargs["_soapheaders"] = soap_headers
45
---> 46 return self._proxy._binding.send(
47 self._proxy._client,
48 self._proxy._binding_options,
/opt/anaconda3/lib/python3.8/site-packages/zeep/wsdl/bindings/soap.py in send(self, client, options, operation, args, kwargs)
133 return response
134
--> 135 return self.process_reply(client, operation_obj, response)
136
137 async def send_async(self, client, options, operation, args, kwargs):
/opt/anaconda3/lib/python3.8/site-packages/zeep/wsdl/bindings/soap.py in process_reply(self, client, operation, response)
179
180 elif response.status_code != 200 and not response.content:
--> 181 raise TransportError(
182 u"Server returned HTTP status %d (no content available)"
183 % response.status_code,
TransportError: Server returned HTTP status 404 (no content available)
Thanks in advance :)

Python Elasticsearch 7.05 indexing failing

I have created an index on my ES server via Kibana that looks like this
PUT export_control
{
"mappings": {
"properties": {
"content": {
"type": "text",
"term_vector": "with_positions_offsets"
}
}
}
}
Result is
{
"acknowledged" : true,
"shards_acknowledged" : true,
"index" : "export_control"
}
When I try to index a new document with the following command in python
col_names = df.columns
for row_number in range(df.shape[0]):
body = dict([(name, str(df.iloc[row_number][name])) for name in col_names])
es.index(index = 'export_control', doc_type="pdf_document", body = body)
I get the following error
RequestError Traceback (most recent call last)
<ipython-input-247-6df868fd60f1> in <module>
2 for row_number in range(df.shape[0]):
3 body = dict([(name, str(df.iloc[row_number][name])) for name in col_names])
----> 4 es.index(index = 'export_control', doc_type="pdf_document", body = body)
/usr/local/lib/python3.7/site-packages/elasticsearch/client/utils.py in _wrapped(*args, **kwargs)
74 for p in es_query_params + GLOBAL_PARAMS:
75 if p in kwargs:
---> 76 v = kwargs.pop(p)
77 if v is not None:
78 params[p] = _escape(v)
/usr/local/lib/python3.7/site-packages/elasticsearch/client/__init__.py in index(self, index, doc_type, body, id, params)
317 "timeout",
318 "timestamp",
--> 319 "ttl",
320 "version",
321 "version_type",
/usr/local/lib/python3.7/site-packages/elasticsearch/transport.py in perform_request(self, method, url, headers, params, body)
316 method = "POST"
317
--> 318 # or as source parameter
319 elif self.send_get_body_as == "source":
320 if params is None:
/usr/local/lib/python3.7/site-packages/elasticsearch/connection/http_urllib3.py in perform_request(self, method, url, params, body, timeout, ignore, headers)
183 kw.update(
184 {
--> 185 "cert_reqs": "CERT_REQUIRED",
186 "ca_certs": ca_certs,
187 "cert_file": client_cert,
/usr/local/lib/python3.7/site-packages/elasticsearch/connection/base.py in _raise_error(self, status_code, raw_data)
123 )
124 logger.debug("> %s", body)
--> 125 logger.debug("< %s", response)
126
127 self._log_trace(method, path, body, status_code, response, duration)
RequestError: TransportError(400, 'illegal_argument_exception', 'Rejecting mapping update to [export_control] as the final mapping would have more than 1 type: [_doc, pdf_document]')
The problem is that in Elasticsearch types were removed recently - some full definition of it, which means, that you could have only one type - called _doc, while you're creating second one - pdf_document.
You should use
es.index(index = 'export_control', doc_type="_doc", body = body)

Resources