Http message is not callable porting python2 to python3 download script - python-3.x

i'am trying to port to python 3 this script :
import re
from os.path import basename
import os
from urllib.parse import urlparse,urlsplit
from urllib.request import urlopen,Request
import urllib
def url2name(url):
return basename(urlsplit(url)[2])
def download(url, out_path="."):
localName = url2name(url)
req = Request(url)
r = urlopen(req)
if r.info().has_key('Content-Disposition'):
# If the response has Content-Disposition, we take file name from it
localName = r.info()['Content-Disposition'].split('filename=')[1]
if localName[0] == '"' or localName[0] == "'":
localName = localName[1:-1]
elif r.url != url:
# if we were redirected, the real file name we take from the final URL
localName = url2name(r.url)
localName = os.path.join(out_path, localName)
f = open(localName, 'wb')
f.write(r.read())
f.close()
but i have a :
'HTTPMessage' object is not callable
r.info() seems to have problems
how to get the header info in python 3 ?

Try with this, you should use context managers:
def download(url, out_path="."):
localName = url2name(url)
req = Request(url)
with urlopen(req) as f:
content_disp = f.getheader('Content-Disposition')
if content_disp:
# If the response has Content-Disposition, we take file name from it
localName = content_disp.split('filename=')[1]
if localName[0] == '"' or localName[0] == "'":
localName = localName[1:-1]
elif f.url != url:
# if we were redirected, the real file name we take from the final URL
localName = url2name(f.url)
localName = os.path.join(out_path, localName)
with open(localName, 'wb') as fp:
fp.write(f.read())

Related

How bind source IP address for http request

i'm trying make http request from specified ip address, but it's doesen't work this way. The required IP address is not assigned during the request .Could someone please help me with this problem ? What i do incorrect ?
import random
import time
import json
import hmac
import hashlib
import requests
from urllib.parse import urljoin, urlencode
from binance import Client
import pandas as pd
import urllib3
def create_order(params): # Func for creating order
print('Creating order')
PATH = '/api/v3/order'
params['timestamp'] = int(time.time() * 1000)
query_string = urlencode(params)
print('Query to: ', query_string)
params['signature'] = hmac.new(secret_Key.encode('utf-8'), query_string.encode('utf-8'), hashlib.sha256).hexdigest()
url = urljoin(BASE_URL, PATH)
real_create_conn = urllib3.util.connection.create_connection
def set_src_addr(address, timeout, *args, **kw):
source_address = ('10.8.5.107', 0)
return real_create_conn(address, timeout=timeout, source_address=source_address)
urllib3.util.connection.create_connection = set_src_addr
r = requests.post(url, headers=headers, params=params)
def Get_Price(params, offset):
PATH = '/api/v3/ticker/price'
params['timestamp'] = int(time.time() * 1000)
params1 = {
'symbol': params['symbol']
}
url = urljoin(BASE_URL, PATH)
r = requests.get(url, headers=headers, params=params1)
if r.status_code == 200:
print(json.dumps(r.json(), indent=2))
else:
print(r)
print(r.status_code, r.json(), '\n')
price = round(float(r.json()['price']), 2)
print('Order Price : ', price + offset, '$')
params['price'] = price + offset
create_order(params)
return params
if __name__ == "__main__":
api_Key = input('Enter API Key:\n')
secret_Key = input('Enter Secret Key:\n')
time_to_sleep = int(input('Enter the frequency of order movement (in milliseconds):\n'))
tryes_limit = int(input('Enter the maximum number of times for a move order :\n'))
symbol = (input('Enter symbol:\n')).upper()
side = (input('Enter side (SELL / BUY):\n')).upper()
offset = float(input('Enter price offset in USD:\n'))
quantity = round(float(input('Enter quantity:\n')), 3)
client = Client(api_Key, secret_Key)
BASE_URL = 'https://api.binance.com'
headers = {
'X-MBX-APIKEY': api_Key
}
params = {
'symbol': symbol,
'side': side,
'type': 'LIMIT',
'timeInForce': 'GTC',
'quantity': quantity,
'price': '',
'recvWindow': 60000,
'timestamp': int()
}
Get_Price(params, offset)
Also i tryed do it by this way :
source = source.SourceAddressAdapter('10.8.5.107')
with requests.Session() as session:
a = session.mount('http://', source)
r = session.post(url, headers=headers, params=params)
print('dasasd', session.post(url, headers=headers, params=params))

How to pull logs from an API and store them in AWS S3

I am trying to achieve this through a Python program. I am able to pull logs from API to a local drive.
However I am struggling to copy them to AWS S3. I appreciate your help on this.
I am using the code below to copy files to a local drive
''''''
import requests
import requests.auth
from bs4 import BeautifulSoup
import os.path
from pathlib import Path
import os
import boto3
CLIENT_ID = "xxxxxxxx"
CLIENT_SECRET = "xxxxxxxx"
TOKEN_URL = "https://xxxxxxxx/dwsso/oauth2/access_token"
REDIRECT_URI = "https://xxxxxxxx/dwsso/oauth2/callback"
def get_token(code=None):
client_auth = requests.auth.HTTPBasicAuth(CLIENT_ID, CLIENT_SECRET)
post_data = {"grant_type": "client_credentials",
"code": code,
"redirect_uri": REDIRECT_URI}
response = requests.post(TOKEN_URL,
auth=client_auth,
data=post_data)
token_json = response.json()
return token_json["access_token"]
my_token=get_token()
headersAPI = {
'accept': 'application/json',
'Authorization': 'Bearer '+ my_token,
}
params = (
('offset', '0'),
('limit', '20'),
)
def download_files(urls):
for a in soup.find_all('a', href=True):
url = urls
a = a['href']
logs_url = url + a
r = requests.get(logs_url, headers=headersAPI, params=params, verify=True, stream = True)
save_path = "download" + ((url.split('/')[2]).split('.')[0])
Path(save_path).mkdir(parents=True, exist_ok=True)
lname = (logs_url.split('/')[-1])
completeName= os.path.join(save_path, lname)
print("Downloding file from %s domain: %s" % ( save_path, lname )
open(completeName, 'wb').write(r.content)
url_lst = [ "https://xxxxxxxx/webdav/Sites/Logs/",
"https://xxxxxxxx/webdav/Sites/Logs/",
"https://xxxxxxxx/Sites/Logs/" ]
for i in url_lst:
response = requests.get(myUrl, headers=headersAPI, params=params, verify=True, stream = True)
soup = BeautifulSoup(response.content, 'html.parser')
f_url = ((i.split('/')[2]))
url = "https://" + f_url
download_files(url)
''''''
You can upload to S3 buckets using Boto3:
import boto3
s3 = boto3.resource('s3')
s3.Bucket('mybucket').upload_file('/tmp/hello.txt', 'hello.txt')
See Upload to S3 Bucket documentation here.
And here for installation and config of Boto3.

Getting Quotes from eTrade API in Python3

I'm trying to get quotes from the etrade API. I'm able to list accounts, get transactions, but not get quotes. I've tried removing the accounts and transactions api calls but it makes no difference. I get an "oauth_problem=signature_invalid" response. Any ideas what I need to do differently?
from rauth import OAuth1Service
import webbrowser
import hmac
# required for google sheets
# from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
class ETradeManager():
def __init__(self):
session = None
service = None
def connect_to_etrade(self):
self.service = OAuth1Service(
name='etrade',
consumer_key='',
consumer_secret='',
request_token_url='https://apisb.etrade.com/oauth/request_token',
access_token_url='https://apisb.etrade.com/oauth/access_token',
authorize_url='https://us.etrade.com/e/t/etws/authorize?key={}&token={}',
base_url='https://etsw.etrade.com')
oauth_token, oauth_token_secret = self.service.get_request_token(params=
{'oauth_callback': 'oob',
'format': 'json'})
auth_url = self.service.authorize_url.format(self.service.consumer_key, oauth_token)
webbrowser.open(auth_url)
verifier = input('Please input the verifier: ')
print("Attempting to get session")
self.session = self.service.get_auth_session(oauth_token, oauth_token_secret, params={'oauth_verifier': verifier})
url = 'https://apisb.etrade.com/v1/accounts/list'
resp = self.session.get(url, params={'format': 'json'})
accountid = ""
print(resp.text)
trans_url_template = "https://apisb.etrade.com/v1/accounts/{}/transactions"
trans_url = trans_url_template.format(accountid)
resp = self.session.get(trans_url, params={'format': 'json'})
f = open("trans.xml", "w")
f.write(resp.text)
# key = self.service.consumer_secret + \
# '&' + \
# oauth_token_secret
# hashed = hmac.new(key.encode(), base_string.encode(), sha1)
# def get_quote(self):
quote_url_template = "https://apisb.etrade.com/v1/market/quote/{}"
quote_url = quote_url_template.format("TSLA")
resp = self.session.get(quote_url_template, params={'format': 'json'})
f = open("quote.xml", "w")
f.write(resp.text)
trade_manager = ETradeManager()
trade_manager.connect_to_etrade()
# trade_manager.get_quote()
Not sure if you figured this out but you had a typo here:
resp = self.session.get(quote_url_template, params={'format': 'json'})
Should be using quote_url, not quote_url_template

File upload using Flask

I am trying to implement a python API in order to upload a file on my server but for an unknown reason, it doesn't run.
From my understanding, the app.py is not recognised
Here is my API.py
from flask_cors import CORS
from flask_restful import Api, Resource, reqparse
import sqlite3
import uuid
import os
import csv
import urllib.request
import threading
import queue as Queue
import subprocess
import json
import re
import datetime
app = Flask(__name__)
api = Api(app)
CORS(app)
class upload(Resource):
def post(self):
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
def allowed_file(filename):
return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
def upload_file():
# check if the post request has the file part
if 'file' not in request.files:
resp = jsonify({'message' : 'No file part in the request'})
resp.status_code = 400
return resp
file = request.files['file']
int = str(request.form['int']) #true or false
if file.filename == '':
resp = jsonify({'message' : 'No file selected for uploading'})
resp.status_code = 400
return resp
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
return json.dumps(data), 200
else:
resp = jsonify({'message' : 'Allowed file types are doc, etc.'})
resp.status_code = 400
return resp
api.add_resource(upload, "/api/v1/upload")
app.run(host='0.0.0.0', debug=True)
Here is my app.py
UPLOAD_FOLDER = '/home/xxxx/xxx/upload'
app = Flask(__name__)
#app.secret_key = "secret key"
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
Can you please help? Any suggestions?
As per the comment, it looks like you have two sepearate applications here.
I would just stick with the first one API.py, but you'll need to move the lines where you set the config variables into API.py:
So after this line in API.py:
app = Flask(__name__)
Immediately set the config values:
UPLOAD_FOLDER = '/home/xxxx/xxx/upload'
#app.secret_key = "something super secret"
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
Then execute with:
python API.py

How to stream video (motion jpeg) using falcon server?

Desired output
Input: Camera feed using OpenCV or from a REST camera url. (Not a concern for this question)
Output: Streaming jpeg images after doing some OpenCV processing
So far I have done the following based on Falcon tutorial
Input: Image file as a POST request
Output: GET request endpoint with the path to the image
import mimetypes
import os
import re
import uuid
import cv2
import io
import falcon
from falcon import media
import json
import msgpack
class Collection(object):
def __init__(self, image_store):
self._image_store = image_store
def on_get(self, req, resp):
# TODO: Modify this to return a list of href's based on
# what images are actually available.
doc = {
'images': [
{
'href': '/images/1eaf6ef1-7f2d-4ecc-a8d5-6e8adba7cc0e.png'
}
]
}
resp.data = msgpack.packb(doc, use_bin_type=True)
resp.content_type = falcon.MEDIA_MSGPACK
resp.status = falcon.HTTP_200
def on_post(self, req, resp):
name = self._image_store.save(req.stream, req.content_type)
# Unnecessary Hack to read the saved file in OpenCV
image = cv2.imread("images/" + name)
new_image = do_something_with_image(image)
_ = cv2.imwrite("images/" + name, new_image)
resp.status = falcon.HTTP_201
resp.location = '/images/' + name
class Item(object):
def __init__(self, image_store):
self._image_store = image_store
def on_get(self, req, resp, name):
resp.content_type = mimetypes.guess_type(name)[0]
resp.stream, resp.stream_len = self._image_store.open(name)
class ImageStore(object):
_CHUNK_SIZE_BYTES = 4096
_IMAGE_NAME_PATTERN = re.compile(
'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.[a-z]{2,4}$'
)
def __init__(self, storage_path, uuidgen=uuid.uuid4, fopen=io.open):
self._storage_path = storage_path
self._uuidgen = uuidgen
self._fopen = fopen
def save(self, image_stream, image_content_type):
ext = mimetypes.guess_extension(image_content_type) # Issue with this code, Not returning the extension so hard coding it in next line
ext = ".jpg"
name = '{uuid}{ext}'.format(uuid=self._uuidgen(), ext=ext)
image_path = os.path.join(self._storage_path, name)
with self._fopen(image_path, 'wb') as image_file:
while True:
chunk = image_stream.read(self._CHUNK_SIZE_BYTES)
if not chunk:
break
image_file.write(chunk)
return name
def open(self, name):
# Always validate untrusted input!
if not self._IMAGE_NAME_PATTERN.match(name):
raise IOError('File not found')
image_path = os.path.join(self._storage_path, name)
stream = self._fopen(image_path, 'rb')
stream_len = os.path.getsize(image_path)
return stream, stream_len
def create_app(image_store):
api = falcon.API()
api.add_route('/images', Collection(image_store))
api.add_route('/images/{name}', Item(image_store))
api.add_sink(handle_404, '')
return api
def get_app():
storage_path = os.environ.get('LOOK_STORAGE_PATH', './images')
image_store = ImageStore(storage_path)
return create_app(image_store)
The response is something like this:
HTTP/1.1 201 Created
Connection: close
Date: Mon, 03 Dec 2018 13:08:14 GMT
Server: gunicorn/19.7.1
content-length: 0
content-type: application/json; charset=UTF-8
location: /images/e69a83ee-b369-47c3-8b1c-60ab7bf875ec.jpg
There are 2 problems with the above code:
I first get the data stream and save it in a file and then read it in OpenCV to do some other operations which is pretty overkill and should be easily fixed but I don't know how
This service does not stream the JPGs. All I have a GET request url which I can open in browser to see the image which is not ideal
So, how can I read req.stream data as numpy array? And more importantly what changes I need to make to stream images from this service?
P.S. Apologies for a long post
I have found a solution which works perfectly fine. For more on this check out this beautiful code.
def gen(camera):
while True:
image = camera.get_frame()
new_image = do_something_with_image(image)
ret, jpeg = cv2.imencode('.jpg', new_image)
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + jpeg.tobytes() + b'\r\n\r\n')
class StreamResource(object):
def on_get(self, req, resp):
labeled_frame = gen(VideoCamera())
resp.content_type = 'multipart/x-mixed-replace; boundary=frame'
resp.stream = labeled_frame
def get_app():
api = falcon.API()
api.add_route('/feed', StreamResource())
return api

Resources