I have the following endpoint in my Flask API:
#api.route("/files", methods=Method.POST)
def upload_file() -> str:
enquirer_id = request.get_json()['id']
try:
logging.info(request.files)
uploaded_file = request.files['image']
except KeyError:
abort(HttpStatusCode.BAD_REQUEST, "Please provide image to upload")
if uploaded_file.content_type != ContentType.JPEG:
abort(HttpStatusCode.BAD_REQUEST, "Only JPEG images are allowed")
filename = str(enquirer_id)
destination_file = os.path.join(current_app.config[AppCfg.DIRECTORY], filename)
uploaded_file.save(destination_file)
return f'Successfully upload file {filename}.'
But I am unable to test this endpoint using the FlaskClient:
class APITestUploadDownload(APITestBase):
url = '/api/v1/files'
my_file = 'some_picture.jpg'
def setUp(self):
self.client = self.app.test_client()
def test_upload(self):
with open(file=self.my_file, mode='rb') as file:
data = {
'image': io.BytesIO(file.read()),
'id': 1
}
response = self.client.post(self.url, data=json.dumps(data), headers={}, content_type='multipart/form-data',)
self.assertEqual(response.status_code, HttpStatusCode.OK)
But this fails with the error message TypeError: Object of type BytesIO is not JSON serializable.
If i make the request without using json.dump(data) using response = self.client.post(self.url, data=data, headers={}, content_type='multipart/form-data', ) I get another error which tells me thatresponse.data is None.
So how can I fix my file upload endpoint and the correspondig test?
Related
Using Python 3.8.10 and requests 2.21.0
The service post handle function below：
from tornado.escape import json_decode
def get_payload(self):
return json_decode(self.request.body)
def post(self):
""" acquire device """
data = self.get_payload()
udid = data["udid"]
idle_timeout = data.get('idleTimeout', 600)
email = self.current_user.email
try:
await D(udid).acquire(email, idle_timeout)
self.write_json({
"success": True,
"description": "Device successfully added"
})
except AcquireError as e:
self.set_status(403) # forbidden
self.write_json({
"success": False,
"description": "Device add failed: " + str(e),
})
The Client post fun below:
def post_occupied_device(self,udid = None,idleTimeout = None):
url = self.SERVER_URL+'/api/v1/user/devices'
occupied_dict = {
"udid": udid
}
try:
resp = requests.post(url,json=json.dumps(occupied_dict),headers=self.headers)
except requests.exceptions.RequestException as e:
print(e)
return "failed,network error"
print(resp)
enter code here
The server return:<Response [500]>
Then i change json data to this ：
resp = requests.post(url,json=json_encode(occupied_dict),headers=self.headers)
The server return:<Response [500]>
Then i change json data to this ：
requests.post(url,json=occupied_dict,headers=self.headers)
it is ok,The server return:<Response [200]>
i checkout the result and type between json_encode and json.dumps：
import json
from tornado.escape import json_encode
occupied_json = {
"udid": "DFEFDGDSGDF"
}
occupied_dict = {
'udid': "DFEFDGDSGDF"
}
req1 = json.dumps(occupied_json)
req2 = json_encode(occupied_json)
print(req1,type(req1))
print(req2,type(req2))
they are the same：
{"udid": "DFEFDGDSGDF"} <class 'str'>
{"udid": "DFEFDGDSGDF"} <class 'str'>
so,why?
I have implemented for our django back-end application (SP) possibility to login via SAML, as IDP im using Keycloak. It works fine, but I want to write tests to be sure that all logic is being executed correctly. For this I want to generate a post request with SAML as body and mock (unittest.mock.patch) the real request. But i stuck.
Here is my django view, which accepts get and post requests when I try to login via SAML:
class SamlLoginView(View):
#staticmethod
def prepare_django_request(request):
if 'HTTP_X_FORWARDED_FOR' in request.META:
server_port = 443
else:
server_port = request.META.get('SERVER_PORT')
result = {
'https': 'on' if request.is_secure() else 'off',
'http_host': request.META['HTTP_HOST'],
'script_name': request.META['PATH_INFO'],
'server_port': server_port,
'get_data': request.GET.copy(),
'post_data': request.POST.copy(),
}
return result
#never_cache
def get(self, *args, **kwargs):
req = SamlLoginView.prepare_django_request(self.request)
auth = OneLogin_Saml2_Auth(req, settings.SAML_IDP_SETTINGS)
return_url = self.request.GET.get('next') or settings.LOGIN_REDIRECT_URL
return HttpResponseRedirect(auth.login(return_to=return_url))
#never_cache
def post(self, *args, **kwargs):
req = SamlLoginView.prepare_django_request(self.request)
print(req['post_data']['SAMLResponse'])
auth = OneLogin_Saml2_Auth(req, settings.SAML_IDP_SETTINGS)
auth.process_response()
errors = auth.get_errors()
if not errors:
if auth.is_authenticated():
logger.info("Login", extra={'action': 'login',
'userid': auth.get_nameid()})
user = authenticate(request=self.request,
saml_authentication=auth)
login(self.request, user)
return HttpResponseRedirect("/")
else:
raise PermissionDenied()
else:
return HttpResponseBadRequest("Error when processing SAML Response: %s" % (', '.join(errors)))
In my tests, I wanted to directly call the post method, in which there will be a saml inside:
class TestSamlLogin(TestCase):
def test_saml_auth(self, prepare):
client = APIClient()
url = reverse_lazy("miri_auth:samllogin")
saml_resp='<xml with saml response>'
resp = client.post(url, data=saml_resp)
but obviously it shows that request.POST is empty.
I then decided to make a mock for the prepare_django_request function, and manually insert the saml:
def mocked_prepare_request(request):
post_query_dict = QueryDict(mutable=True)
post_data = {
'SAMLResponse': saml_xml,
'RelayState': '/accounts/profile/'
}
post_query_dict.update(post_data)
result = {
'https': 'on',
'http_host': '<http-host>',
'script_name': '/api/auth/samllogin/',
'server_port': '443',
'get_data': {},
'post_data': post_query_dict,
}
return result
class TestSamlLogin(TestCase):
#patch('miri_auth.views.SamlLoginView.prepare_django_request', side_effect=mocked_prepare_request)
def test_saml_auth(self, prepare):
client = APIClient()
url = reverse_lazy("miri_auth:samllogin")
saml_resp='<xml with saml response>'
resp = client.post(url, data=saml_resp)
and depending on how I pass the saml_xml it throws different errors, if i define it as string:
with open(os.path.join(TEST_FILES_PATH, 'saml.xml')) as f:
saml_xml = " ".join([x.strip() for x in f])
it returns: lxml.etree.XMLSyntaxError: Start tag expected, '<' not found, line 1, column 1, although I checked the output from saml_xml in the xml validator and it says that the xml is valid.
When i try to parse the file into xml in advance, i get another error later,
libraries with which I tried to parse:
import xml.etree.ElementTree as ET
from xml.dom import minidom
from lxml import etree
tree = etree.parse(os.path.join(TEST_FILES_PATH, 'saml.xml'))
it returns:
TypeError: argument should be a bytes-like object or ASCII string, not '_ElementTree'
Debugging these errors didn't lead me to any solution.
If anyone has any thoughts on how this can be implemented (Mocking response with SAML), or where I made a mistake, I would be glad to hear.
Thank in advance
I realized that the SAML Response must be encoded:
with open(os.path.join(TEST_FILES_PATH, 'saml.xml')) as f:
saml_xml = " ".join([x.strip() for x in f])
base64_saml = base64.b64encode(saml_xml.encode('ascii')).decode('ascii')
post_data = {'SAMLResponse': base64_saml, 'RelayState': '/accounts/profile/'}
url = reverse_lazy("miri_auth:samllogin")
request = self.client.post(url, post_data)
but now i am getting the following errors:
func=xmlSecOpenSSLEvpDigestVerify:file=digests.c:line=280:obj=sha256:subj=unknown:error=12:invalid data:data and digest do not match
I'm struggling to reassemble an image sent via a POST request to a GCP Cloud Function.
I've looked at advice here about how to package a file with POST request.
I would like the function to reconstruct the image from bytes for further processing, everytime I send the request I get 'Failure' back. Any help would be really appreciated!
### client_side.py
import requests
url = 'https://region-project.cloudfunctions.net/function' # Generic GCP functions URL
files = {'file': ('my_image.jpg', open('my_image.jpg', 'rb').read(), 'application/octet-stream')}
r = requests.post(url, files=files)
### gcp_function.py
from io import BytesIO
def handler(request):
try:
incoming = request.files.get('file')
bytes = BytesIO(incoming)
image = open_image(bytes)
message = 'Success'
except:
message = 'Failure'
return message
Sorted it.
Needed the read method to convert FileStorage Object to bytes.
### gcp_function.py
from io import BytesIO
import logging
def handler(request):
try:
incoming = request.files['file'].read()
bytes = BytesIO(incoming)
image = open_image(bytes)
message = 'Success'
except Exception as e:
message = 'Failure'
logging.critical(str(e))
return message
Desired output
Input: Camera feed using OpenCV or from a REST camera url. (Not a concern for this question)
Output: Streaming jpeg images after doing some OpenCV processing
So far I have done the following based on Falcon tutorial
Input: Image file as a POST request
Output: GET request endpoint with the path to the image
import mimetypes
import os
import re
import uuid
import cv2
import io
import falcon
from falcon import media
import json
import msgpack
class Collection(object):
def __init__(self, image_store):
self._image_store = image_store
def on_get(self, req, resp):
# TODO: Modify this to return a list of href's based on
# what images are actually available.
doc = {
'images': [
{
'href': '/images/1eaf6ef1-7f2d-4ecc-a8d5-6e8adba7cc0e.png'
}
]
}
resp.data = msgpack.packb(doc, use_bin_type=True)
resp.content_type = falcon.MEDIA_MSGPACK
resp.status = falcon.HTTP_200
def on_post(self, req, resp):
name = self._image_store.save(req.stream, req.content_type)
# Unnecessary Hack to read the saved file in OpenCV
image = cv2.imread("images/" + name)
new_image = do_something_with_image(image)
_ = cv2.imwrite("images/" + name, new_image)
resp.status = falcon.HTTP_201
resp.location = '/images/' + name
class Item(object):
def __init__(self, image_store):
self._image_store = image_store
def on_get(self, req, resp, name):
resp.content_type = mimetypes.guess_type(name)[0]
resp.stream, resp.stream_len = self._image_store.open(name)
class ImageStore(object):
_CHUNK_SIZE_BYTES = 4096
_IMAGE_NAME_PATTERN = re.compile(
'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.[a-z]{2,4}$'
)
def __init__(self, storage_path, uuidgen=uuid.uuid4, fopen=io.open):
self._storage_path = storage_path
self._uuidgen = uuidgen
self._fopen = fopen
def save(self, image_stream, image_content_type):
ext = mimetypes.guess_extension(image_content_type) # Issue with this code, Not returning the extension so hard coding it in next line
ext = ".jpg"
name = '{uuid}{ext}'.format(uuid=self._uuidgen(), ext=ext)
image_path = os.path.join(self._storage_path, name)
with self._fopen(image_path, 'wb') as image_file:
while True:
chunk = image_stream.read(self._CHUNK_SIZE_BYTES)
if not chunk:
break
image_file.write(chunk)
return name
def open(self, name):
# Always validate untrusted input!
if not self._IMAGE_NAME_PATTERN.match(name):
raise IOError('File not found')
image_path = os.path.join(self._storage_path, name)
stream = self._fopen(image_path, 'rb')
stream_len = os.path.getsize(image_path)
return stream, stream_len
def create_app(image_store):
api = falcon.API()
api.add_route('/images', Collection(image_store))
api.add_route('/images/{name}', Item(image_store))
api.add_sink(handle_404, '')
return api
def get_app():
storage_path = os.environ.get('LOOK_STORAGE_PATH', './images')
image_store = ImageStore(storage_path)
return create_app(image_store)
The response is something like this:
HTTP/1.1 201 Created
Connection: close
Date: Mon, 03 Dec 2018 13:08:14 GMT
Server: gunicorn/19.7.1
content-length: 0
content-type: application/json; charset=UTF-8
location: /images/e69a83ee-b369-47c3-8b1c-60ab7bf875ec.jpg
There are 2 problems with the above code:
I first get the data stream and save it in a file and then read it in OpenCV to do some other operations which is pretty overkill and should be easily fixed but I don't know how
This service does not stream the JPGs. All I have a GET request url which I can open in browser to see the image which is not ideal
So, how can I read req.stream data as numpy array? And more importantly what changes I need to make to stream images from this service?
P.S. Apologies for a long post
I have found a solution which works perfectly fine. For more on this check out this beautiful code.
def gen(camera):
while True:
image = camera.get_frame()
new_image = do_something_with_image(image)
ret, jpeg = cv2.imencode('.jpg', new_image)
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + jpeg.tobytes() + b'\r\n\r\n')
class StreamResource(object):
def on_get(self, req, resp):
labeled_frame = gen(VideoCamera())
resp.content_type = 'multipart/x-mixed-replace; boundary=frame'
resp.stream = labeled_frame
def get_app():
api = falcon.API()
api.add_route('/feed', StreamResource())
return api
Getting django.utils.datastructures.MultiValueDictKeyError: 'email_id' while post request. Here's my code
views.py:
def post(self, request):
video_file = self.request.data['media_file']
user_email_id = self.request.data['email_id']
file_storage = FileSystemStorage()
saved_file_name = file_storage.save(str(uuid.uuid4()), video_file)
upload.py:
headers = {"Authorizarion": "auth/key"}
data = {"email_id": "email#email.com",
"media_file": open(video.file_path, "rb")}
response = requests.post("/api/url", headers=headers, data=data)
When try to upload the file via Postman works fin as it takes care of Content-Type, but trying to upload in the backend makes it difficult to clear it.
It seems you're not passing the email_id - can you print its value in upload.py before you post?
To be on the safe side, provide a default value in your views.py:
video_file = self.request.data.get("media_file", None)
user_email_id = self.request.data.get("email_id", None)
Replace None with whatever you want as a default if a field is missing.
UPDATE: Of course, then you'll encounter the issue of uploading file contents the way you do. To upload the actual file you should use a files structure in your requests.post() call, e.g.:
headers = {"Authorizarion": "auth/key"}
data = {"email_id": "email#email.com"}
files = {"media_file": open(video.file_path, "rb")}
response = requests.post("/api/url", headers=headers, data=data, files=files)
Then on Django side you'll have to retrieve the contents as:
def post(self, request):
video_file = request.FILES.get('media_file', None)
user_email_id = request.POST.get('email_id', None)
# you should probably validate video_file and user_email_id here
file_storage = FileSystemStorage()
saved_file_name = file_storage.save(str(uuid.uui`d4()), video_file)