I am trying to read tweets having specific keywords using docker. I have taken reference from
Github link .
I have made some minor changes. While I'm trying to execute I am facing issues with a number of arguments through all the details in place. It would be great if anybody can guide me where I'm doing wrong
### twitter
import tweepy
from tweepy.auth import OAuthHandler
from tweepy import Stream
#from tweepy.streaming import StreamListener
import json
import logging
### logging
FORMAT = "%(asctime)s | %(name)s - %(levelname)s - %(message)s"
LOG_FILEPATH = "C:\\docker-kafka\\log\\testing.log"
logging.basicConfig(
filename=LOG_FILEPATH,
level=logging.INFO,
filemode='w',
format=FORMAT)
### Authenticate to Twitter
with open('C:\\docker-kafka\\credential.json','r') as f:
credential = json.load(f)
CONSUMER_KEY = credential['twitter_api_key']
CONSUMER_SECRET = credential['twitter_api_secret_key']
ACCESS_TOKEN = credential['twitter_access_token']
ACCESS_TOKEN_SECRET = credential['twitter_access_token_secret']
BEARER_TOKEN = credential['bearer_token']
#from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from kafka import KafkaProducer
producer = KafkaProducer(bootstrap_servers='localhost:9092',
value_serializer=lambda v: v.encode('utf-8')) #Same port as your Kafka server
topic_name = "docker-twitter"
class twitterAuth():
"""SET UP TWITTER AUTHENTICATION"""
def authenticateTwitterApp(self):
auth = OAuthHandler(consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
return auth
class TwitterStreamer():
"""SET UP STREAMER"""
def __init__(self):
self.twitterAuth = twitterAuth()
def stream_tweets(self):
while True:
listener = ListenerTS()
auth = self.twitterAuth.authenticateTwitterApp()
stream = Stream(auth, listener)
stream.filter(track=["Starbucks"], stall_warnings=True, languages= ["en"])
class ListenerTS(tweepy.Stream):
def on_status(self, status):
tweet = json.dumps({
'id': status.id,
'text': status.text,
'created_at': status.created_at.strftime("%Y-%m-%d %H:%M:%S")
}, default=str)
producer.send(topic_name, tweet)
return True
if __name__ == "__main__":
TS = TwitterStreamer()
TS.stream_tweets()
Answer reference:
Not able to read from Tweets from twitter
As far as i understand, the class tweepy.Stream needs to be initialized, even if inheriting it. So, instead of:
try
class ListenerTS(tweepy.Stream):
def __init__(self):
tweepy.Stream.__init__(self, CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
Also check this docs:
https://docs.tweepy.org/en/stable/streaming.html
And maybe this link:
https://improveandrepeat.com/2022/04/python-friday-117-streaming-search-results-with-tweepy/
I was able to resolve the issue by adding the secrets but got a different error.
def stream_tweets(self):
while True:
listener = ListenerTS(CONSUMER_KEY,CONSUMER_SECRET,ACCESS_TOKEN,ACCESS_TOKEN_SECRET)
listener.filter(track=["Starbucks"], stall_warnings=True, languages= ["en"])
Related
I am building a Restful API in Python 3.8 using flask_restful, flask_apispec, and marshmallow. When I enable use_kwargs decorator my app is crushing on a POST with TypeError: post() takes 1 positional argument but 2 were given
I'd appreciate any help in solving this bug.
# requirements.txt
aniso8601==9.0.1
apispec==5.1.1
certifi==2021.10.8
charset-normalizer==2.0.7
click==8.0.3
Flask==2.0.2
flask-apispec==0.11.0
Flask-RESTful==0.3.9
gunicorn==20.1.0
idna==3.3
itsdangerous==2.0.1
Jinja2==3.0.2
MarkupSafe==2.0.1
marshmallow==3.14.0
pytz==2021.3
requests==2.26.0
six==1.16.0
tableauserverclient==0.17.0
urllib3==1.26.7
webargs==8.0.1
Werkzeug==2.0.2
from apispec import APISpec
from flask import Flask, request
from flask_restful import Resource, Api
from apispec.ext.marshmallow import MarshmallowPlugin
from flask_apispec.extension import FlaskApiSpec
from marshmallow import Schema, fields, post_load, ValidationError
from flask_apispec.views import MethodResource
from flask_apispec import use_kwargs, marshal_with
app = Flask(__name__) # Flask app instance initiated
api = Api(app) # Flask restful wraps Flask app around it.
app.config.update({
'APISPEC_SPEC': APISpec(
title='Kube Controller',
version='v1',
plugins=[MarshmallowPlugin()],
openapi_version='2.0.0'
),
'APISPEC_SWAGGER_URL': '/swagger/', # URI to access API Doc JSON
'APISPEC_SWAGGER_UI_URL': '/swagger-ui/' # URI to access UI of API Doc
})
docs = FlaskApiSpec(app)
class AwesomeRequestSchema(Schema):
api_type = fields.String(required=True)
#post_load
def create(self, data, **kwargs):
return MyAPI(**data)
class MyAPI:
def __init__(self, api_type):
self.api_type = api_type
self.message = "hi"
class AwesomeAPI(MethodResource, Resource):
#use_kwargs(AwesomeRequestSchema)
#marshal_with(AwesomeRequestSchema, code=200, description='Something created')
def post(self):
"""
POST
"""
try:
schema = AwesomeRequestSchema()
data = schema.load(request.json)
print(data.api_type)
return request.json
except ValidationError as err:
return err.messages
api.add_resource(AwesomeAPI, '/')
docs.register(AwesomeAPI)
if __name__ == '__main__':
app.run(debug=True)
Thanks!
I had the same issue. The use_kwargs decorator will try to populate and inject the AwesomeRequestSchema object into the post() method: https://flask-apispec.readthedocs.io/en/latest/api_reference.html#flask_apispec.annotations.use_kwargs
To fix, replace
def post(self):
with
def post(self, populated_request_object):
I'm trying to get quotes from the etrade API. I'm able to list accounts, get transactions, but not get quotes. I've tried removing the accounts and transactions api calls but it makes no difference. I get an "oauth_problem=signature_invalid" response. Any ideas what I need to do differently?
from rauth import OAuth1Service
import webbrowser
import hmac
# required for google sheets
# from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
class ETradeManager():
def __init__(self):
session = None
service = None
def connect_to_etrade(self):
self.service = OAuth1Service(
name='etrade',
consumer_key='',
consumer_secret='',
request_token_url='https://apisb.etrade.com/oauth/request_token',
access_token_url='https://apisb.etrade.com/oauth/access_token',
authorize_url='https://us.etrade.com/e/t/etws/authorize?key={}&token={}',
base_url='https://etsw.etrade.com')
oauth_token, oauth_token_secret = self.service.get_request_token(params=
{'oauth_callback': 'oob',
'format': 'json'})
auth_url = self.service.authorize_url.format(self.service.consumer_key, oauth_token)
webbrowser.open(auth_url)
verifier = input('Please input the verifier: ')
print("Attempting to get session")
self.session = self.service.get_auth_session(oauth_token, oauth_token_secret, params={'oauth_verifier': verifier})
url = 'https://apisb.etrade.com/v1/accounts/list'
resp = self.session.get(url, params={'format': 'json'})
accountid = ""
print(resp.text)
trans_url_template = "https://apisb.etrade.com/v1/accounts/{}/transactions"
trans_url = trans_url_template.format(accountid)
resp = self.session.get(trans_url, params={'format': 'json'})
f = open("trans.xml", "w")
f.write(resp.text)
# key = self.service.consumer_secret + \
# '&' + \
# oauth_token_secret
# hashed = hmac.new(key.encode(), base_string.encode(), sha1)
# def get_quote(self):
quote_url_template = "https://apisb.etrade.com/v1/market/quote/{}"
quote_url = quote_url_template.format("TSLA")
resp = self.session.get(quote_url_template, params={'format': 'json'})
f = open("quote.xml", "w")
f.write(resp.text)
trade_manager = ETradeManager()
trade_manager.connect_to_etrade()
# trade_manager.get_quote()
Not sure if you figured this out but you had a typo here:
resp = self.session.get(quote_url_template, params={'format': 'json'})
Should be using quote_url, not quote_url_template
First of all, I have tried looking for answers in this website. But no luck...
What I wanna achieve is that starting an independent thread in the request handling thread to do some asynchronous task. The tricky point is that there are some database operations needed in this independent thread.
Here is an example. Five files included.
project
|__manager.py
|__config.py
|__deployer
|__`__init__.py`
|__models.py
|__views.py
|__operators.py
Detail code below...
# deployer/__init__.py
from flask import Flask
from deployer.models import db
def create_app():
app = Flask(__name__)
app.config.from_object(object_name)
db.init_app(app)
# Add route for index
#app.route('/')
def index():
return {'code': 200, 'message': 'OK'}
return app
# manager.py
from os import environ
from flask_script import Manager, Server
from deployer import create_app
from flask_restful import Api
from deployer.views import HostView
env = environ.get('APM_ENV', 'dev')
app = create_app('config.%sConfig' % env.capitalize())
api = Api(app)
api.add_resource(HostView, '/api/v1/hosts')
manager = Manager(app)
manager.add_command("server", Server(host='0.0.0.0', port=9527))
if __name__ == '__main__':
manager.run(default_command='server')
# deployer/views.py
from flask_restful import Resource, reqparse
from flask import jsonify
from deployer.models import db, Host
from deployer.operators import HostInitiator
parser = reqparse.RequestParser()
parser.add_argument('host', type=int, help='Specify an unique host.')
class HostView(Resource):
def get(self):
h = db.session.query(Host).filter(Host.id == 1).one()
return jsonify(
host_id=h.id,
host_code=h.code,
host_ip=h.ip_addr_v4
)
def post(self):
h = Host(
code='Harbor',
ip_addr_v4='10.10.10.199',
state='created'
)
db.session.add(h)
db.session.commit()
initiator = HostInitiator(host=h)
initiator.start()
return {
'code': 'Harbor',
'ip_addr_v4': '10.10.10.199',
'state': 'created'
}
# deployer/models.py
from sqlalchemy import Column, Integer, String
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
class Host(db.Model):
__tablename__ = 'br_host'
id = Column(Integer, primary_key=True, autoincrement=True)
code = Column(String(128), index=True, nullable=False)
ip_addr_v4 = Column(String(15), nullable=False)
state = Column(String(16), nullable=False)
# deployer/operators.py
from threading import Thread
from deployer.models import db
class HostInitiator(Thread):
def __init__(self, host):
super().__init__()
self.host = host
def run(self):
# Update Host.state [created-->initating]
db.session.query(Host).filter(Host.id == self.host.id).update({'state': 'initating'})
db.session.commit()
# do some initiating things...
# Update Host.state [initating-->ready]
db.session.query(Host).filter(Host.id == self.host.id).update({'state': 'ready'})
db.session.commit()
Always got outside application context error with code above. The error message indicates that no database operation is permitted in the HostInitiator thread.
It suggests me to push a context or move my code into a view function. I'm suffering this quite a while, please help out if you guys have any suggestions. Thanks in advance.
The code works for me
def test_multi_threading_query():
# module which i create Flask app instance
from app.main import app
# module which i create sqlalchemhy instance
from app.model.db import db, Post
with app.app_context():
posts = Post.query.all()
p = posts[0]
p.foo = 1
db.session.add(p)
db.session.commit()
print(p)
#api.route('/test')
def test_view():
from threading import Thread
t = Thread(target=test_multi_threading_query)
t.start()
return ''
# main.py
app = Flask(__main__)
#db.py
db = SQLAlchemy()
class Post(db.Model):
id = db.Column(db.Integer, primary_key=True)
foo = db.Column(db.Integer)
https://flask.palletsprojects.com/en/1.1.x/appcontext/
import tornado.web
import tornado.ioloop
from apiApplicationModel import userData
from cleanArray import Label_Correction
import json
import requests
colName=['age', 'resting_blood_pressure', 'cholesterol', 'max_heart_rate_achieved', 'st_depression', 'num_major_vessels', 'st_slope_downsloping', 'st_slope_flat', 'st_slope_upsloping', 'sex_male', 'chest_pain_type_atypical angina', 'chest_pain_type_non-anginal pain', 'chest_pain_type_typical angina', 'fasting_blood_sugar_lower than 120mg/ml', 'rest_ecg_left ventricular hypertrophy', 'rest_ecg_normal', 'exercise_induced_angina_yes', 'thalassemia_fixed defect', 'thalassemia_normal',
'thalassemia_reversable defect']
class processRequestHandler(tornado.web.RequestHandler):
def post(self):
data_input_array = []
for name in colName:
x = self.get_body_argument(name, default=0)
data_input_array.append(int(x))
label = Label_Correction(data_input_array)
finalResult = int(userData(label))
output = json.dumps({"Giveput":finalResult})
self.write(output)
class basicRequestHandler(tornado.web.RequestHandler):
def get(self):
self.render('report.html')
class staticRequestHandler(tornado.web.RequestHandler):
def post(self):
data_input_array = []
for name in colName:
x = self.get_body_argument(name, default=0)
data_input_array.append(str(x))
send_data = dict(zip(colName, data_input_array))
print(send_data)
print(type(send_data))
url = "http://localhost:8887/output"
headers={}
response = requests.request('POST',url,headers=headers,data=send_data)
print(response.text.encode('utf8'))
print("DONE")
if __name__=='__main__':
app = tornado.web.Application([(r"/",basicRequestHandler),
(r"/result",staticRequestHandler),
(r"/output",processRequestHandler)])
print("API IS RUNNING.......")
app.listen(8887)
tornado.ioloop.IOLoop.current().start()
Actually I am trying to create an API and the result of the API can be used but
The page keeps on loading, and no response is shown.
Response should be a python dictionary send by post function of class processRequestHandler
After using a debugger the lines after response = requests.request('POST',url,headers=headers,data=send_data) are not executed.
The class processRequestHandler is working fine when checked with POSTMAN.
requests.request is a blocking method. This blocks the event loop and prevents any other handlers from running. In a Tornado handler, you need to use Tornado's AsyncHTTPClient (or another non-blocking HTTP client such as aiohttp) instead.
async def post(self):
...
response = await AsyncHTTPClient().fetch(url, method='POST', headers=headers, body=send_data)
See the Tornado users's guide for more information.
I am doing Twitter feed sentiment analysis based on a keyword but I keep getting Error 406.
Mostly the error is coming in sentiment_Analysis function because I checked removing that and I get no error.
Please find my code below:
from __future__ import absolute_import, print_function
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
import urllib.request
import urllib.parse
import urllib.error
consumer_key='qr****************T1' #consumer key
consumer_secret='ZbkO**********************U9Y' #consumer secret
access_token='989***************************2kd' #access token
access_token_secret='Fv****************************qY7' #access secret
#For sentiment Analysis we will be using Sentigem API key
akhil_auth= "297**********************************hB"
def sentiment_Analysis(text):
encoded_text=urllib.parse.quote(text)
#API_Call= "https://api.sentigem.com/external/get-sentiment?api-key="+akhil_auth+"&text="+text
# output=urllib.urlopen(API_Call).read
output = urllib.request.urlopen(API_Call)
return output
class StdOutListener(StreamListener):
def on_data(self,data):
tweet= data.split(',"text":"')[1].split('","source')[0]
sentimentRating=sentiment_Analysis(tweet)
saveTweets= tweet+'::'+sentimentRating+'\n'
output=open('output.csv','a')
output.write(saveTweets)
output.close()
print("Conversion successful.")
return True
def on_error(self, status):
print (status)
if __name__ == '__main__':
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
#topic = input("Read tweets cotaining: ")
stream.filter("Burger King")