File upload using Flask - python-3.x

I am trying to implement a python API in order to upload a file on my server but for an unknown reason, it doesn't run.
From my understanding, the app.py is not recognised
Here is my API.py
from flask_cors import CORS
from flask_restful import Api, Resource, reqparse
import sqlite3
import uuid
import os
import csv
import urllib.request
import threading
import queue as Queue
import subprocess
import json
import re
import datetime
app = Flask(__name__)
api = Api(app)
CORS(app)
class upload(Resource):
def post(self):
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
def allowed_file(filename):
return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
def upload_file():
# check if the post request has the file part
if 'file' not in request.files:
resp = jsonify({'message' : 'No file part in the request'})
resp.status_code = 400
return resp
file = request.files['file']
int = str(request.form['int']) #true or false
if file.filename == '':
resp = jsonify({'message' : 'No file selected for uploading'})
resp.status_code = 400
return resp
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
return json.dumps(data), 200
else:
resp = jsonify({'message' : 'Allowed file types are doc, etc.'})
resp.status_code = 400
return resp
api.add_resource(upload, "/api/v1/upload")
app.run(host='0.0.0.0', debug=True)
Here is my app.py
UPLOAD_FOLDER = '/home/xxxx/xxx/upload'
app = Flask(__name__)
#app.secret_key = "secret key"
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
Can you please help? Any suggestions?

As per the comment, it looks like you have two sepearate applications here.
I would just stick with the first one API.py, but you'll need to move the lines where you set the config variables into API.py:
So after this line in API.py:
app = Flask(__name__)
Immediately set the config values:
UPLOAD_FOLDER = '/home/xxxx/xxx/upload'
#app.secret_key = "something super secret"
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
Then execute with:
python API.py

Related

Unittest : How test upload file function with request.file

I am new at unittest and I want test Function that upload an .ond file but I still don't know how can I use request.file correctly to upload file and send in test : here is my code and test code
def upload_ond_file():
if 'ondFile' not in request.files:
return 'No file'
directory = 'src/BDDs/BDD_Onduleurs'
if not os.path.isdir(path + directory):
os.mkdir(path + directory)
file = request.files['ondFile']
file.save(os.path.join(path + directory, file.filename))
The test was like :
import unittest
from urllib import response
from flask_unittest import ClientTestCase
from requests import request
import src.Controller.InverterController as InverterController
from unittest import mock
import io
class SomefileTestCase(unittest.TestCase):
def test_method_called_from_route(self):
m = mock.MagicMock()
request.method = "POST"
testfile
=open('C:/optisolar/optisolarbe/src/Test/uploads/test_controller_inverter/20170131_SC 2500-EV_MS_V6.ond',"r")
data= testfile.read()
data = {'files[]': testfile}
with mock.patch("src.Controller.InverterController.request", m):
result = InverterController.upload_ond_file()
print(result)
if __name__ == '__main__':
unittest.main()
but when execute code it mentione that there is no file :
request.url = <MagicMock name='mock.url' id='1807586099888'>
request.path = <MagicMock name='mock.path' id='1807586136320'>
request.data = <MagicMock name='mock.data' id='1807586164448'>
request.method = <MagicMock name='mock.method' id='1807586192640'>
request.files = <MagicMock name='mock.files' id='1807586224928'>
request.filesname = <MagicMock name='mock.files.name' id='1807586253120'>
request.filespath = <MagicMock name='mock.files.path' id='1807586289360'>
request.values = <MagicMock name='mock.values' id='1807586317600'>
request.args = <MagicMock name='mock.args' id='1807586345984'>
request.form = <MagicMock name='mock.form' id='1807586374176'>
request.getjson = <MagicMock name='mock.get_json()' id='1807586422544'>
No file
C:\Python39\lib\unittest\case.py:550: ResourceWarning: unclosed file <_io.TextIOWrapper name='C:/optisolar/optisolarbe/src/Test/uploads/test_controller_inverter/20170131_SC 2500-EV_MS_V6.ond' mode='r' encoding='cp1252'>
method()
ResourceWarning: Enable tracemalloc to get the object allocation traceback
.
----------------------------------------------------------------------
Ran 1 test in 0.025s
OK
my question is how can I upload file (request.file) correctly and send it in test ! thank you in advance

Getting Quotes from eTrade API in Python3

I'm trying to get quotes from the etrade API. I'm able to list accounts, get transactions, but not get quotes. I've tried removing the accounts and transactions api calls but it makes no difference. I get an "oauth_problem=signature_invalid" response. Any ideas what I need to do differently?
from rauth import OAuth1Service
import webbrowser
import hmac
# required for google sheets
# from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
class ETradeManager():
def __init__(self):
session = None
service = None
def connect_to_etrade(self):
self.service = OAuth1Service(
name='etrade',
consumer_key='',
consumer_secret='',
request_token_url='https://apisb.etrade.com/oauth/request_token',
access_token_url='https://apisb.etrade.com/oauth/access_token',
authorize_url='https://us.etrade.com/e/t/etws/authorize?key={}&token={}',
base_url='https://etsw.etrade.com')
oauth_token, oauth_token_secret = self.service.get_request_token(params=
{'oauth_callback': 'oob',
'format': 'json'})
auth_url = self.service.authorize_url.format(self.service.consumer_key, oauth_token)
webbrowser.open(auth_url)
verifier = input('Please input the verifier: ')
print("Attempting to get session")
self.session = self.service.get_auth_session(oauth_token, oauth_token_secret, params={'oauth_verifier': verifier})
url = 'https://apisb.etrade.com/v1/accounts/list'
resp = self.session.get(url, params={'format': 'json'})
accountid = ""
print(resp.text)
trans_url_template = "https://apisb.etrade.com/v1/accounts/{}/transactions"
trans_url = trans_url_template.format(accountid)
resp = self.session.get(trans_url, params={'format': 'json'})
f = open("trans.xml", "w")
f.write(resp.text)
# key = self.service.consumer_secret + \
# '&' + \
# oauth_token_secret
# hashed = hmac.new(key.encode(), base_string.encode(), sha1)
# def get_quote(self):
quote_url_template = "https://apisb.etrade.com/v1/market/quote/{}"
quote_url = quote_url_template.format("TSLA")
resp = self.session.get(quote_url_template, params={'format': 'json'})
f = open("quote.xml", "w")
f.write(resp.text)
trade_manager = ETradeManager()
trade_manager.connect_to_etrade()
# trade_manager.get_quote()
Not sure if you figured this out but you had a typo here:
resp = self.session.get(quote_url_template, params={'format': 'json'})
Should be using quote_url, not quote_url_template

sending compressed numpy array (zlib) to flask server with post request [python]

I am trying to send a compressed numpy array (compressed with zlib) to the flask server with post request, but the compressed bytes object is getting changed in the server end. How to properly send the bytes object with requests post request so that I can decompress on the server end?
server.py
from flask import Flask
from flask_restful import Resource, Api, reqparse
import json
import numpy as np
import base64
# compression
import zlib
app = Flask(__name__)
api = Api(app)
parser = reqparse.RequestParser()
parser.add_argument('imgb64')
class Predict(Resource):
def post(self):
data = parser.parse_args()
if data['imgb64'] == "":
return {
'data':'',
'message':'No file found',
'status':'error'
}
img = data['imgb64']
print('rec')
# decompress
print(type(img))
print(img)
dec = zlib.decompress(img) # this gives me error
if img:
pass
return json.dumps({
'data': 'done',
'message':'darknet processed',
'status':'success'
})
return {
'data':'',
'message':'Something when wrong',
'status':'error'
}
api.add_resource(Predict,'/predict')
if __name__ == '__main__':
app.run(debug=True, host = '0.0.0.0', port = 5000, threaded=True)
client.py
import numpy as np
import base64
import zlib
import requests
frame = np.random.randint(0,255,(5,5,3)) # dummy rgb image
# compress
data = zlib.compress(frame)
print('b64 encoded')
print(data)
print(len(data))
print(type(data))
r = requests.post("http://127.0.0.1:5000/predict", data={'imgb64' : data}) # sending compressed numpy array
This gives me the following error:
TypeError: a bytes-like object is required, not 'str'
So, I tried to convert the string to bytes object:
dec = zlib.decompress(img.encode()) # this gives me error
But, this one also gives me an error:
zlib.error: Error -3 while decompressing data: incorrect header check
I tried with other encodings, they also failed.
One thing I noticed is, when I print the compressed bytes in the client end, it reads:
b'x\x9c-\xcf?J\x82q\x00\x06\xe0\x0ftrVP\\C\xf4\x06\x9268\t\xcdR\x8ej.F\xa0\xe0\xd0\xa6\xa3\xe0V\x07\x10\x1cL\xc8\xd1\x03\xd4\xe4\t\x0c\x12\x84\xb6D\x0c#\xbc\x80O\xf0\x1b\x9e\xf5\xfdS\x89\xa2h\xcf\x9a\x03\xef\xc4\xf8cF\x92\r\xbf4i\x11g\xc83\x0f\x8c\xb9\xa2#\x8e\x1bn\xd91g\xc0\x91%\xd7\xdc\xf3M\x83<i:L\xa8\xf1\x19\xfa\xffw\xfd\xf0\xc5\x94:O\x9cH\x85\xcc6#\x1e\xc3\xf6\x05\xe5\xa0\xc7\x96\x04]J\\\x90\xa1\x1f~Ty\xe1\x8d\x15w|P\xe4\x95K\xb2!\xe3\x0cw)%I'
But on the server end, the received string is completely different:
�4ig�3���#�n�1g��%���M�<i:L����w��Ŕ:O�H��6#���ǖ]J\��~Ty�w|P�K�!�w)%I
I also tried to send the bytes as string, by
r = requests.post("http://127.0.0.1:5000/predict", data={'imgb64' : str(data)})
But, I can't decompress the data on the server end.
It seems, I can't send the zlib compressed bytes directly, so I used base64 to encode the data into ascii string.
So, in summary this worked for me, numpy array/any non-string data -> zlib compression -> base64 encode -> post request -> flask -> base64 decode -> zlib decompress
client.py
import numpy as np
import base64
import zlib
import requests
frame = np.random.randint(0,255,(5,5,3)) # dummy rgb image
# compress
data = zlib.compress(frame)
data = base64.b64encode(data)
data_send = data
data2 = base64.b64decode(data)
data2 = zlib.decompress(data2)
fdata = np.frombuffer(data2, dtype=np.uint8)
print(fdata)
r = requests.post("http://127.0.0.1:5000/predict", data={'imgb64' : data_send})
server.py
from flask import Flask
from flask_restful import Resource, Api, reqparse
import json
import numpy as np
import base64
# compression
import zlib
import codecs
app = Flask(__name__)
api = Api(app)
parser = reqparse.RequestParser()
parser.add_argument('imgb64', help = 'type error')
class Predict(Resource):
def post(self):
data = parser.parse_args()
#print(data)
if data['imgb64'] == "":
return {
'data':'',
'message':'No file found',
'status':'error'
}
#img = open(data['imgb64'], 'r').read() # doesn't work
img = data['imgb64']
data2 = img.encode()
data2 = base64.b64decode(data2)
data2 = zlib.decompress(data2)
fdata = np.frombuffer(data2, dtype=np.uint8)
print(fdata)
if img:
return json.dumps({
'data': 'done',
'message':'darknet processed',
'status':'success'
})
return {
'data':'',
'message':'Something when wrong',
'status':'error'
}
api.add_resource(Predict,'/predict')
if __name__ == '__main__':
app.run(debug=True, host = '0.0.0.0', port = 5000, threaded=True)

How to set the path while working on images in Flask on localhost as well as ibm cloud

This application was running fine till yesterday but today it isn't working. No changes have been made in backend or front end. The site Output is like this. I am trying to use open cv to extract 4 sub images from a image which has been uploaded in ../static/uploads folder. While the extracted 4 images are stored in ../static/extracted folder. Now my concern is, yesterday the code was running but today it isn't working. And while deploying to the ibm cloud, I could not find open cv. "cv2 module not found". Can someone please help me improve this code.
app.py code:
from cloudant import Cloudant
from flask import Flask, render_template, request, jsonify
import atexit
import os
import json
from ocr_core import ocr_core
UPLOAD_FOLDER = '/static/uploads/'
ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif'])
app = Flask(__name__,static_url_path='')
db_name = 'mydb'
client = None
db = None
if 'VCAP_SERVICES' in os.environ:
vcap = json.loads(os.getenv('VCAP_SERVICES'))
print('Found VCAP_SERVICES')
if 'cloudantNoSQLDB' in vcap:
creds = vcap['cloudantNoSQLDB'][0]['credentials']
user = creds['username']
password = creds['password']
url = 'https://' + creds['host']
client = Cloudant(user, password, url=url, connect=True)
db = client.create_database(db_name, throw_on_exists=False)
elif "CLOUDANT_URL" in os.environ:
client = Cloudant(os.environ['CLOUDANT_USERNAME'], os.environ['CLOUDANT_PASSWORD'], url=os.environ['CLOUDANT_URL'], connect=True)
db = client.create_database(db_name, throw_on_exists=False)
elif os.path.isfile('vcap-local.json'):
with open('vcap-local.json') as f:
vcap = json.load(f)
print('Found local VCAP_SERVICES')
creds = vcap['services']['cloudantNoSQLDB'][0]['credentials']
user = creds['username']
password = creds['password']
url = 'https://' + creds['host']
client = Cloudant(user, password, url=url, connect=True)
db = client.create_database(db_name, throw_on_exists=False)
# On IBM Cloud Cloud Foundry, get the port number from the environment variable PORT
# When running this app on the local machine, default the port to 8000
port = int(os.getenv('PORT', 5000))
def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
#app.route('/')
def home_page():
return render_template('index.html')
#app.route('/upload', methods=['GET', 'POST'])
def upload_page():
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
return render_template('upload.html', msg='No file selected')
file = request.files['file']
if file.filename == '':
return render_template('upload.html', msg='No file selected')
if file and allowed_file(file.filename):
file.save(os.path.join(os.getcwd() + UPLOAD_FOLDER, file.filename))
extracted_text = ocr_core(file)
return render_template('upload.html',
msg='Successfully processed',
extracted_text=extracted_text,
img_src=UPLOAD_FOLDER + file.filename)
elif request.method == 'GET':
return render_template('upload.html')
if __name__ == '__main__':
app.run(host='0.0.0.0', port=port, debug=True)
ocr_py.py:
try:
from PIL import Image
except ImportError:
import Image
import pytesseract
import cv2
from random import randint
import pandas as pd
#UPLOAD_FOLDER = '/static'
def ocr_core(filename):
#text = pytesseract.image_to_string(Image.open(filename))
val=str(filename)
val=list(val)
naam=""
tt=0
for i in range(15,len(val)):
if(val[i]!="'"):
naam+=val[i]
if(val[i]=='g'):
tt=1
if(tt==1):
break
image = cv2.imread("/home/sahil/CheckCheque-deploy/static/uploads/" +str(naam))
k=randint(0, 999999)
cropped1 = image[290:500, 320:1540]
cv2.imwrite("/home/sahil/CheckCheque-deploy/static/extracted/name"+str(k)+".png", cropped1)
name=pytesseract.image_to_string(Image.open("/home/sahil/CheckCheque-deploy/static/extracted/name"+str(k)+".png"))
cropped2 = image[470:700, 670:2640]
cv2.imwrite("/home/sahil/CheckCheque-deploy/static/extracted/amount"+str(k)+".png", cropped2)
amount=pytesseract.image_to_string(Image.open("/home/sahil/CheckCheque-deploy/static/extracted/amount"+str(k)+".png"))
cropped3 = image[850:1000, 480:1040]
cv2.imwrite("/home/sahil/CheckCheque-deploy/static/extracted/acc_no"+str(k)+".png", cropped3)
acc_no=pytesseract.image_to_string(Image.open("/home/sahil/CheckCheque-deploy/static/extracted/acc_no"+str(k)+".png"))
cropped5 = image[500:850, 2940:4500]
cv2.imwrite("/home/sahil/CheckCheque-deploy/static/extracted/amt_num"+str(k)+".png", cropped5)
amt_num=pytesseract.image_to_string(Image.open("/home/sahil/CheckCheque-deploy/static/extracted/amt_num"+str(k)+".png"))
acc_no1=""
lnum=["1","0","2","3","4","5","6","7","8","9"]
for i in range(0,len(acc_no)):
if(str(acc_no[i]) in lnum):
acc_no1+=acc_no[i]
l=[name,acc_no,amt_num]
df = pd.read_csv("/home/sahil/CheckCheque-deploy/jobchahiye.csv")
df.loc[df.Account== int(l[1]), 'Amount'] -=int(l[2])
df.loc[df.Name== str(l[0]), 'Amount'] +=int(l[2])
df.to_csv("/home/sahil/CheckCheque-deploy/jobchahiye.csv", index=False)
return l
Looks like you have a dependency on either opencv-python or opencv-contrib-python. As such it needs to be installed as a pre-requisite on any machine that you want to run your application.
On localhost you do this with pip, but if you are not running something like virtualenv then all installs will be to the global space, and conflicts can easily arise, resulting in you not being able to run anything.
For cloud deployments you specify pip prerequisites in your requirements.txt file

Unable to read the buffer from BytesIO in google app engine flex environment

Here is the related code
import logging
logging.getLogger('googleapicliet.discovery_cache').setLevel(logging.ERROR)
import datetime
import json
from flask import Flask, render_template, request
from flask import make_response
from googleapiclient.discovery import build
from googleapiclient.http import MediaIoBaseDownload
from oauth2client.client import AccessTokenCredentials
...
#app.route('/callback_download')
def userselectioncallback_with_drive_api():
"""
Need to make it a background process
"""
logging.info("In download callback...")
code = request.args.get('code')
fileId = request.args.get('fileId')
logging.info("code %s", code)
logging.info("fileId %s", fileId)
credentials = AccessTokenCredentials(
code,
'flex-env/1.0')
http = httplib2.Http()
http_auth = credentials.authorize(http)
# Exports a Google Doc to the requested MIME type and returns the exported content. Please note that the exported content is limited to 10MB.
# v3 does not work? over quota?
drive_service = build('drive', 'v3', http=http_auth)
drive_request = drive_service.files().export(
fileId=fileId,
mimeType='application/pdf')
b = bytes()
fh = io.BytesIO(b)
downloader = MediaIoBaseDownload(fh, drive_request)
done = False
try:
while done is False:
status, done = downloader.next_chunk()
logging.log("Download %d%%.", int(status.progress() * 100))
except Exception as err:
logging.error(err)
logging.error(err.__class__)
response = make_response(fh.getbuffer())
response.headers['Content-Type'] = 'application/pdf'
response.headers['Content-Disposition'] = \
'inline; filename=%s.pdf' % 'yourfilename'
return response
It is based on some code example of drive api. I am trying to export some files from google drive to pdf format.
The exception comes from the line
response = make_response(fh.getbuffer())
It throws the exception:
TypeError: 'memoryview' object is not callable
How can I retrieve the pdf content properly from the fh? Do I need to further apply some base 64 encoding?
My local runtime is python 3.4.3
I have used an incorrect API. I should do this instead:
response = make_response(fh.getvalue())

Resources