I have a synchronous code with requests that I am trying to move to using aiohttp.ClientSession.
Indeed, I have a class in which I set a aiohttp.ClientSession with various headers, among those an API key. The above code works for requesting data: (I deleted the init.... everything works but this function)
class Client():
def __init__(self):
loop = asyncio.get_event_loop()
self.session = aiohttp.ClientSession(
loop=loop,
headers=self.get_headers()
)
def send_signed_request(self, url_path, payload={}):
session = requests.session()
session.headers.update(self._get_headers())
query_string = urlencode(payload)
url = url_path + '?' + query_string
params = {'url': url, 'params': {}}
response = session.post(**params)
return response.json()
client = Client()
results = client.send_signed_request(url, params)
From that, with the requests session, I obtain a valid response from the server.
for some reason, the code below, with aiohttp session does not work and I have no idea how to adapt it.
async def send_signed_request(self, url_path, payload={}):
query_string = urlencode(payload)
url = url_path + '?' + query_string
params = {'url': url, 'data': {}}
async with self.session.post(**params) as response:
return await response.json()
does anybody knows my error please?
Related
Trying to work through this signed request example script with no success.
I notice both "http_method" and "params" in "def send_signed_request" are both greyed out to indicate unused code.
I have tried to add the params to the request as below, but that's not going to solve the unused code issue, both are still showing as unused code.
response = send_signed_request('POST', '/sapi/v1/margin/loan', {"asset": "", "isIsolated": "TRUE", "symbol": "", "amount": ""} )
print(response)
I'm just learning Python and maybe missing some assumed knowledge I guess and have been reading a lot to no avail before posting.
I read somewhere that Binance are seeing lots of traders spending hours trying to solve signature authentication as I am, and maybe this will help others in the same situation.
thx in advance to anyone that takes a look.
Just to clarify the script didn't come from Binance and I would have to dig for the link if anyone wants it.
import hmac
import time
import hashlib
import requests
from urllib.parse import urlencode
KEY = ''
SECRET = ''
BASE_URL = 'https://sapi.binance.com' # production base url
#BASE_URL = 'https://testnet.binancefuture.com' # testnet base url
''' ====== begin of functions, you don't need to touch ====== '''
def hashing(query_string):
return hmac.new(SECRET.encode('utf-8'), query_string.encode('utf-8'), hashlib.sha256).hexdigest()
def get_timestamp():
return int(time.time() * 1000)
def dispatch_request(http_method):
session = requests.Session()
session.headers.update({
'Content-Type': 'application/json;charset=utf-8',
'X-MBX-APIKEY': KEY
})
return {
'GET': session.get,
'DELETE': session.delete,
'PUT': session.put,
'POST': session.post,
}.get(http_method, 'GET')
# used for sending request requires the signature
def send_signed_request(http_method, url_path, payload={}):
query_string = urlencode(payload)
url = BASE_URL + url_path + '?' + query_string + '&signature=' + hashing(query_string)
params = {'url': url, 'params': {}}
# used for sending public data request
# def send_public_request(url_path, payload={}):
# query_string = urlencode(payload, True)
# url = BASE_URL + url_path
# if query_string:
# url = url + '?' + query_string
# print("{}".format(url))
# response = dispatch_request('GET')(url=url)
# return response.json()
response = send_signed_request('POST', '/sapi/v1/margin/loan', params )
print(response)
Below is a working script if anyone has the same issue
import hmac
import time
import hashlib
import requests
from urllib.parse import urlencode
KEY = ''
SECRET= ''
BASE_URL = 'https://api.binance.com' # production base url
#BASE_URL = 'https://testnet.binancefuture.com' # testnet base url
''' ====== begin of functions, you don't need to touch ====== '''
def hashing(query_string):
return hmac.new(SECRET.encode('utf-8'), query_string.encode('utf-8'), hashlib.sha256).hexdigest()
def get_timestamp():
return int(time.time() * 1000)
def dispatch_request(http_method):
session = requests.Session()
session.headers.update({
'Content-Type': 'application/json;charset=utf-8',
'X-MBX-APIKEY': KEY
})
return {
'GET': session.get,
'DELETE': session.delete,
'PUT': session.put,
'POST': session.post,
}.get(http_method, 'GET')
# used for sending request requires the signature
def send_signed_request(http_method, url_path, payload={}):
query_string = urlencode(payload)
# replace single quote to double quote
query_string = query_string.replace('%27', '%22')
if query_string:
query_string = "{}×tamp={}".format(query_string, get_timestamp())
else:
query_string = 'timestamp={}'.format(get_timestamp())
url = BASE_URL + url_path + '?' + query_string + '&signature=' + hashing(query_string)
print("{} {}".format(http_method, url))
params = {'url': url, 'params': {}}
response = dispatch_request(http_method)(**params)
print(response)
return response.json()
# used for sending public data request
def send_public_request(url_path, payload={}):
query_string = urlencode(payload, True)
url = BASE_URL + url_path
if query_string:
url = url + '?' + query_string
print("{}".format(url))
response = dispatch_request('GET')(url=url)
return response.json()
#response = send_signed_request('GET', '/api/v3/time')
#print(response)
response = send_signed_request('POST', '/sapi/v1/margin/loan', {"asset": "SHIB", "isIsolated": "TRUE", "symbol": "SHIBUSDT", "amount": "1000.00"} )
print(response)
I'm using AWS Elasticsearch and async elasticsearch-py package in my project to connect with the cluster.
AWS Elasticsearch Version 7.9
Python package: elasticsearch[async]==7.12.0
I'm not able to initialize the Async Elasticsearch client using the AWS4Auth library (mentioned in official AWS ES client Python documentation)
It should successfully connect with the client. However, it gives me this error:
AttributeError: 'AWS4Auth' object has no attribute 'encode'
Sharing my code snippet:
from elasticsearch import AsyncElasticsearch, AIOHttpConnection
from requests_aws4auth import AWS4Auth
import asyncio
host = 'my-test-domain.us-east-1.es.amazonaws.com'
region = 'us-east-1'
service = 'es'
credentials = {
'access_key': "MY_ACCESS_KEY",
'secret_key': "MY_SECRET_KEY"
}
awsauth = AWS4Auth(credentials['access_key'], credentials['secret_key'], region, service)
es = AsyncElasticsearch(
hosts=[{'host': host, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=AIOHttpConnection
)
async def test():
print(await es.info())
asyncio.run(test())
class AWSAuthAIOHttpConnection(AIOHttpConnection):
"""Enable AWS Auth with AIOHttpConnection for AsyncElasticsearch
The AIOHttpConnection class built into elasticsearch-py is not currently
compatible with passing AWSAuth as the `http_auth` parameter, as suggested
in the docs when using AWSAuth for the non-async RequestsHttpConnection class:
https://docs.aws.amazon.com/opensearch-service/latest/developerguide/request-signing.html#request-signing-python
To work around this we patch `AIOHttpConnection.perform_request` method to add in
AWS Auth headers before making each request.
This approach was synthesized from
* https://stackoverflow.com/questions/38144273/making-a-signed-http-request-to-aws-elasticsearch-in-python
* https://github.com/DavidMuller/aws-requests-auth
* https://github.com/jmenga/requests-aws-sign
* https://github.com/byrro/aws-lambda-signed-aiohttp-requests
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._credentials = boto3.Session().get_credentials()
async def perform_request(
self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None
):
def _make_full_url(url: str) -> str:
# These steps are copied from the parent class' `perform_request` implementation.
# The ElasticSearch client only passes in the request path as the url,
# and that partial url format is rejected by the `SigV4Auth` implementation
if params:
query_string = urlencode(params)
else:
query_string = ""
full_url = self.host + url
full_url = self.url_prefix + full_url
if query_string:
full_url = "%s?%s" % (full_url, query_string)
return full_url
full_url = _make_full_url(url)
if headers is None:
headers = {}
# this request object won't be used, we just want to copy its auth headers
# after `SigV4Auth` processes it and adds the headers
_request = AWSRequest(
method=method, url=full_url, headers=headers, params=params, data=body
)
SigV4Auth(self._credentials, "es", "us-west-1").add_auth(_request)
headers.update(_request.headers.items())
# passing in the original `url` param here works too
return await super().perform_request(
method, full_url, params, body, timeout, ignore, headers
)
I took #francojposa's answer above and fixed/adapted it, I tried to submit an edit to his answer but the "suggestion queue is full" or such
requirements.txt:
boto3<2.0
elasticsearch[async]<7.14 # in 7.14 they "shut-out" anything other than elastic cloud
And here's the main definition
from urllib.parse import urlencode
import boto3
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from elasticsearch import AsyncElasticsearch, AIOHttpConnection
class AWSAuthAIOHttpConnection(AIOHttpConnection):
"""Enable AWS Auth with AIOHttpConnection for AsyncElasticsearch
The AIOHttpConnection class built into elasticsearch-py is not currently
compatible with passing AWSAuth as the `http_auth` parameter, as suggested
in the docs when using AWSAuth for the non-async RequestsHttpConnection class:
https://docs.aws.amazon.com/opensearch-service/latest/developerguide/request-signing.html#request-signing-python
To work around this we patch `AIOHttpConnection.perform_request` method to add in
AWS Auth headers before making each request.
This approach was synthesized from
* https://stackoverflow.com/questions/38144273/making-a-signed-http-request-to-aws-elasticsearch-in-python
* https://github.com/DavidMuller/aws-requests-auth
* https://github.com/jmenga/requests-aws-sign
* https://github.com/byrro/aws-lambda-signed-aiohttp-requests
"""
def __init__(self, *args, aws_region=None, **kwargs):
super().__init__(*args, **kwargs)
self.aws_region = aws_region
self._credentials = boto3.Session().get_credentials()
self.auther = SigV4Auth(self._credentials, "es", self.aws_region)
def _make_full_url(self, url: str, params=None) -> str:
# These steps are copied from the parent class' `perform_request` implementation.
# The ElasticSearch client only passes in the request path as the url,
# and that partial url format is rejected by the `SigV4Auth` implementation
query_string = urlencode(params) if params else None
full_url = self.url_prefix + self.host + url
if query_string:
full_url = "%s?%s" % (full_url, query_string)
return full_url
async def perform_request(
self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None
):
full_url = self._make_full_url(url)
if headers is None:
headers = {}
# this request object won't be used, we just want to copy its auth headers
# after `SigV4Auth` processes it and adds the headers
_request = AWSRequest(
method=method, url=full_url, headers=headers, params=params, data=body
)
self.auther.add_auth(_request)
headers.update(_request.headers.items())
# passing in the original `url` param here works too
return await super().perform_request(
method, url, params, body, timeout, ignore, headers
)
Usage:
es_client = AsyncElasticsearch(
['https://aws-es-or-opensearch-url-goes-here'],
use_ssl=True, verify_certs=True,
connection_class=AWSAuthAIOHttpConnection, aws_region='us-east-1'
)
async def test():
body = {...}
results = await es_client.search(body=body, index='test', doc_type='test') # I use ES 5/6
I think that with AWS4Auth you are bound to RequestsHttpConnection.
The default connection class is based on urllib3 which is more
performant and lightweight than the optional requests-based class.
Only use RequestsHttpConnection if you have need of any of requests
advanced features like custom auth plugins etc.
from https://elasticsearch-py.readthedocs.io/en/master/transports.html
Try:
es = AsyncElasticsearch(
hosts=[{'host': host, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection
)
or non-async version if the code above doesn't work:
es = Elasticsearch(
hosts=[{'host': host, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection
)
I have implemented for our django back-end application (SP) possibility to login via SAML, as IDP im using Keycloak. It works fine, but I want to write tests to be sure that all logic is being executed correctly. For this I want to generate a post request with SAML as body and mock (unittest.mock.patch) the real request. But i stuck.
Here is my django view, which accepts get and post requests when I try to login via SAML:
class SamlLoginView(View):
#staticmethod
def prepare_django_request(request):
if 'HTTP_X_FORWARDED_FOR' in request.META:
server_port = 443
else:
server_port = request.META.get('SERVER_PORT')
result = {
'https': 'on' if request.is_secure() else 'off',
'http_host': request.META['HTTP_HOST'],
'script_name': request.META['PATH_INFO'],
'server_port': server_port,
'get_data': request.GET.copy(),
'post_data': request.POST.copy(),
}
return result
#never_cache
def get(self, *args, **kwargs):
req = SamlLoginView.prepare_django_request(self.request)
auth = OneLogin_Saml2_Auth(req, settings.SAML_IDP_SETTINGS)
return_url = self.request.GET.get('next') or settings.LOGIN_REDIRECT_URL
return HttpResponseRedirect(auth.login(return_to=return_url))
#never_cache
def post(self, *args, **kwargs):
req = SamlLoginView.prepare_django_request(self.request)
print(req['post_data']['SAMLResponse'])
auth = OneLogin_Saml2_Auth(req, settings.SAML_IDP_SETTINGS)
auth.process_response()
errors = auth.get_errors()
if not errors:
if auth.is_authenticated():
logger.info("Login", extra={'action': 'login',
'userid': auth.get_nameid()})
user = authenticate(request=self.request,
saml_authentication=auth)
login(self.request, user)
return HttpResponseRedirect("/")
else:
raise PermissionDenied()
else:
return HttpResponseBadRequest("Error when processing SAML Response: %s" % (', '.join(errors)))
In my tests, I wanted to directly call the post method, in which there will be a saml inside:
class TestSamlLogin(TestCase):
def test_saml_auth(self, prepare):
client = APIClient()
url = reverse_lazy("miri_auth:samllogin")
saml_resp='<xml with saml response>'
resp = client.post(url, data=saml_resp)
but obviously it shows that request.POST is empty.
I then decided to make a mock for the prepare_django_request function, and manually insert the saml:
def mocked_prepare_request(request):
post_query_dict = QueryDict(mutable=True)
post_data = {
'SAMLResponse': saml_xml,
'RelayState': '/accounts/profile/'
}
post_query_dict.update(post_data)
result = {
'https': 'on',
'http_host': '<http-host>',
'script_name': '/api/auth/samllogin/',
'server_port': '443',
'get_data': {},
'post_data': post_query_dict,
}
return result
class TestSamlLogin(TestCase):
#patch('miri_auth.views.SamlLoginView.prepare_django_request', side_effect=mocked_prepare_request)
def test_saml_auth(self, prepare):
client = APIClient()
url = reverse_lazy("miri_auth:samllogin")
saml_resp='<xml with saml response>'
resp = client.post(url, data=saml_resp)
and depending on how I pass the saml_xml it throws different errors, if i define it as string:
with open(os.path.join(TEST_FILES_PATH, 'saml.xml')) as f:
saml_xml = " ".join([x.strip() for x in f])
it returns: lxml.etree.XMLSyntaxError: Start tag expected, '<' not found, line 1, column 1, although I checked the output from saml_xml in the xml validator and it says that the xml is valid.
When i try to parse the file into xml in advance, i get another error later,
libraries with which I tried to parse:
import xml.etree.ElementTree as ET
from xml.dom import minidom
from lxml import etree
tree = etree.parse(os.path.join(TEST_FILES_PATH, 'saml.xml'))
it returns:
TypeError: argument should be a bytes-like object or ASCII string, not '_ElementTree'
Debugging these errors didn't lead me to any solution.
If anyone has any thoughts on how this can be implemented (Mocking response with SAML), or where I made a mistake, I would be glad to hear.
Thank in advance
I realized that the SAML Response must be encoded:
with open(os.path.join(TEST_FILES_PATH, 'saml.xml')) as f:
saml_xml = " ".join([x.strip() for x in f])
base64_saml = base64.b64encode(saml_xml.encode('ascii')).decode('ascii')
post_data = {'SAMLResponse': base64_saml, 'RelayState': '/accounts/profile/'}
url = reverse_lazy("miri_auth:samllogin")
request = self.client.post(url, post_data)
but now i am getting the following errors:
func=xmlSecOpenSSLEvpDigestVerify:file=digests.c:line=280:obj=sha256:subj=unknown:error=12:invalid data:data and digest do not match
I would simply like to associate responses from aiohttp asynchronous HTTP requests with an identifier. I am using the following code to hit the API and extract contactproperty object which requires an external field (contacid) in order to call its API:
def get_contact_properties(self, office_name, api_key, ids, chunk_size=100, **params):
properties_pages = []
batch = 0
while True:
chunk_ids = [ids[i] for i in range(batch * chunk_size + 1, chunk_size * (1 + batch) + 1)]
urls = ["{}/{}".format(self.__get_base_url(), "contacts/{}/properties?api_key={}".format(contactid, api_key))
for contactid in chunk_ids]
responses_raw = self.get_responses(urls, self.get_office_token(office_name), chunk_size)
try:
responses_json = [json.loads(response_raw) for response_raw in responses_raw]
except Exception as e:
print(e)
valid_responses = self.__get_valid_contact_properties_responses(responses_json)
properties_pages.append(valid_responses)
if len(valid_responses) < chunk_size: # this is how we know there are no more pages with data
break
else:
batch = batch + 1
ids is a list of ids. The problem is that I do not know which response corresponds to which id so that later I can link it to contact entity using contacid. This is my fetch() function so I was wondering how to edit this function to return the contactid along with output.
async def __fetch(self, url, params, session):
async with session.get(url, params=params) as response:
output = await response.read()
return (output)
async def __bound_fetch(self, sem, url, params, session):
# Getter function with semaphore.
async with sem:
output = await self.__fetch(url, params, session)
return output
You can return the url (or whatever key identifies your request) together with the output.
Regarding using the data, I think you should read the response directly as JSON, especially since aiohttp can do this for you automatically.
async def __fetch(self, url, params, session):
async with session.get(url, params=params) as response:
try:
data = await response.json()
except ValueError as exc:
print(exc)
return None
return data
async def __bound_fetch(self, sem, url, params, session):
# Getter function with semaphore.
async with sem:
output = await self.__fetch(url, params, session)
return {"url": url, "data": data}
You did not post the get_responses function but I'm guessing something like this should work:
responses = self.get_responses(urls, self.get_office_token(office_name), chunk_size)
Responses will be a list of {"url": url, data: "data"} (data can be None for invalid responses); however with the code above one invalid request will not affect the others.
I have 2 URLs and 60k+ requests. Basically, I need to post every request to both URLs, then compare their responses, but not to wait for the response to post another request.
I've tried to do it with aiohttp and asyncio
import asyncio
import time
import aiohttp
import os
from aiofile import AIOFile
testURL = ""
prodURL = ""
directoryWithRequests = ''
directoryToWrite = ''
headers = {'content-type': 'application/soap+xml'}
i = 1
async def fetch(session, url, reqeust):
global i
async with session.post(url=url, data=reqeust.encode('utf-8'), headers=headers) as response:
if response.status != 200:
async with AIOFile(directoryToWrite + str(i) + '.xml', 'w') as afp:
await afp.write(reqeust)
i += 1
return await response.text()
async def fetch_all(session, urls, request):
results = await asyncio.gather(*[asyncio.create_task(fetch(session, url, request)) for url in urls])
return results
async def asynchronousRequests(requestBody):
urls = [testURL, prodURL]
global i
with open(requestBody) as my_file:
body = my_file.read()
async with aiohttp.ClientSession() as session:
htmls = await fetch_all(session, urls, body)
# some conditions
async def asynchronous():
try:
start = time.time()
futures = [asynchronousRequests(directoryWithRequests + i) for i in os.listdir(directoryWithRequests)]
for future in asyncio.as_completed(futures):
result = await future
print("Process took: {:.2f} seconds".format(time.time() - start))
except Exception as e:
print(str(e))
if __name__ == '__main__':
try:
# AsyncronTest
ioloop = asyncio.ProactorEventLoop()
ioloop.run_until_complete(asynchronous())
ioloop.close()
if i == 1:
print('Regress is OK')
else:
print('Number of requests to check = {}'.format(i))
except Exception as e:
print(e)
I believe that the code above works, but it creates N futures, where the N equals to the number of request files. This brings to sort of ddos because the server can't response to that number of requests at the same time.
Found suitable solution. Basically it's just 2 async tasks:
tasks = [
postRequest(testURL, client, body),
postRequest(prodURL, client, body)
]
await asyncio.wait(tasks)
It's not the same performance as the code in the question with afortable number of requests, but as least it doesn't ddos the server that much.