Reddit and Twitter bot in Python using PRAW - python-3.x

I am a beginner in Python, and trying out making a bot which automatically Tweets anything which is posted on a Subreddit that I have made.
I took help from some of the tutorials online which has the following code
import praw
import json
import requests
import tweepy
import time
access_token = '************************************'
access_token_secret = '************************************'
consumer_key = '************************************'
consumer_secret = '************************************'
def strip_title(title):
if len(title) == 94:
return title
else:
return title[:93] + "..."
def tweet_creator(subreddit_info):
post_dict = {}
post_ids = []
print("[bot] Getting posts from Reddit")
for submission in subreddit_info.get_hot(limit=20):
post_dict[strip_title(submission.title)] = submission.url
post_ids.append(submission.id)
print("[bot] Generating short link using goo.gl")
mini_post_dict = {}
for post in post_dict:
post_title = post
post_link = post_dict[post]
short_link = shorten(post_link)
mini_post_dict[post_title] = short_link
return mini_post_dict, post_ids
def setup_connection_reddit(subreddit):
print("[bot] setting up connection with Reddit")
r = praw.Reddit(' %s' %(subreddit))
subreddit = r.get_subreddit(subreddit)
return subreddit
def shorten(url):
headers = {'content-type': 'application/json'}
payload = {"longUrl": url}
url = "https://www.googleapis.com/urlshortener/v1/url"
r = requests.post(url, data=json.dumps(payload), headers=headers)
link = json.loads(r.text)['id']
return link
def duplicate_check(id):
found = 0
with open('posted_posts.txt', 'r') as file:
for line in file:
if id in line:
found = 1
return found
def add_id_to_file(id):
with open('posted_posts.txt', 'a') as file:
file.write(str(id) + "\n")
def main():
subreddit = setup_connection_reddit('*Name of the subreddit*')
post_dict, post_ids = tweet_creator(subreddit)
tweeter(post_dict, post_ids)
def tweeter(post_dict, post_ids):
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
for post, post_id in zip(post_dict, post_ids):
found = duplicate_check(post_id)
if found == 0:
print("[bot] Posting this link on twitter")
print(post + " " + post_dict[post] + " #Python #reddit #bot")
api.update_status(post+" "+post_dict[post]+" #Python #reddit #bot")
add_id_to_file(post_id)
time.sleep(30)
else:
print("[bot] Already posted")
if __name__ == '__main__':
main()
The code seems fine in PyCharm, however I am getting the following error when I try to run it directly from the folder via Terminal using the rolling code, reddit_bot2.py is my file name:
python3 reddit_bot2.py
When I try to run the code I am getting the following error:
mahesh#Maheshs-MacBook-Air Atoms % python3 reddit_bot2.py
[bot] setting up connection with Reddit
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/configparser.py", line 846, in items
d.update(self._sections[section])
KeyError: '**Name of the subreddit to fetch posts from**'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/mahesh/Python_Bot/Atoms/reddit_bot2.py", line 82, in <module>
main()
File "/Users/mahesh/Python_Bot/Atoms/reddit_bot2.py", line 62, in main
subreddit = setup_connection_reddit('Bot167')
File "/Users/mahesh/Python_Bot/Atoms/reddit_bot2.py", line 36, in setup_connection_reddit
r = praw.Reddit(' %s' %(subreddit))
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/praw/reddit.py", line 227, in __init__
self.config = Config(
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/praw/config.py", line 85, in __init__
self.custom = dict(Config.CONFIG.items(site_name), **settings)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/configparser.py", line 849, in items
raise NoSectionError(section)
configparser.NoSectionError: No section: ' Bot167'
You provided the name of a praw.ini configuration which does not exist.
For help with creating a Reddit instance, visit
https://praw.readthedocs.io/en/latest/code_overview/reddit_instance.html
For help on configuring PRAW, visit
https://praw.readthedocs.io/en/latest/getting_started/configuration.html
Any help in this regards would be highly appreciated.
Thanks :)

Related

Python script for Parsing Office 365 HTML email and getting body

This is script to read email and parse body from outlook 365. For me does not work.
I have outlook 365 account without TFA. TFA is not activated
what could be a problem?
#!/usr/bin/python3
from email.message import EmailMessage
import email
import imaplib
import re
import sys
import logging
import base64
import email.parser
import html2text
import requests
import json
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-mpass', '-mailbox_password', dest = 'mailbox_password', help = 'mailbox password.')
args = parser.parse_args()
user = 'someuser#company.com'
mailbox_password ='Email Password' #args.mailbox_password
def get_email_body(body):
if body.is_multipart():
for payload in body.get_payload():
print('To:\t\t', body['To'])
print('From:\t', body['From'])
print('Subject:', body['Subject'])
print('Date:\t',body['Date'])
for part in body.walk():
if (part.get_content_type() == 'text/plain') and (part.get('Content-Disposition') is None):
output = part.get_payload()
else:
print('To:\t\t', body['To'])
print('From:\t', body['From'])
print('Subject:', body['Subject'])
print('Date:\t', body['Date'])
print('Thread-Index:\t', body['Thread-Index'])
text = f"{body.get_payload(decode=True)}"
html = text.replace("b'", "")
h = html2text.HTML2Text()
h.ignore_links = True
output = (h.handle(f'''{html}''').replace("\\r\\n", ""))
output = output.replace("'", "")
# output in one line
#output = output.replace('\n'," ")
output = output.replace('*', "")
return output
def clear_inbox(conn, dest_folder):
output=[]
result = conn.uid('COPY', emailid, dest_folder)
output.append(result)
if result[0] == 'OK':
result = mov, data = conn.uid('STORE',emailid, '+FLAGS', '(\Deleted Items)')
conn.expunge()
conn = imaplib.IMAP4_SSL("outlook.office365.com")
conn.login(user,mailbox_password)
conn.select("Inbox")
try:
resp, items = conn.uid("search",None, 'All')
items = items[0].split()
for emailid in items:
resp, data = conn.uid("fetch",emailid, "(RFC822)")
if resp == 'OK':
email_body = data[0][1].decode('utf-8')
email_message = email.message_from_string(email_body)
subject = email_message["Subject"]
if subject.lower().startswith('Darktrace'.lower()):
output = get_email_body(email_message)
#do some task
# move emails to Processed folder and clear Inbox
clear_inbox(conn, "Processed")
else:
clear_inbox(conn, "backup")
except IndexError:
print("No new email")
conn.close()
conn.logout()
This is error:
Traceback (most recent call last):
File "d:\python\lib\imaplib.py", line 1047, in _command_complete
typ, data = self._get_tagged_response(tag, expect_bye=logout)
File "d:\python\lib\imaplib.py", line 1173, in _get_tagged_response
self._get_response()
File "d:\python\lib\imaplib.py", line 1075, in _get_response
resp = self._get_line()
File "d:\python\lib\imaplib.py", line 1185, in _get_line
raise self.abort('socket error: EOF')
abort: socket error: EOF
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Dev Codes\ReadOutlookEmail.py", line 74, in
resp, data = conn.uid("fetch", emailid, "(RFC822)")
File "d:\python\lib\imaplib.py", line 890, in uid
typ, dat = self._simple_command(name, command, *args)
File "d:\python\lib\imaplib.py", line 1230, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "d:\python\lib\imaplib.py", line 1049, in _command_complete
raise self.abort('command: %s => %s' % (name, val))
abort: command: UID => socket error: EOF

Python Django LineBot Error 54 'Connection reset by peer'

I'm trying to crete linebot through python Django, I want to send some messages and let it scrape the website. Since there is form on the website, I use post request to send the form.
Although I scrape the data successfully, there is error message showed in the python. It seems linebot occupy post method in Django and I send another post request again. I'm not sure my understanding is correct. I can't find any solution about this. Could someone teach me how to fix it?
Exception happened during processing of request from ('127.0.0.1', 50246)
Traceback (most recent call last):
File "/usr/local/anaconda3/lib/python3.8/socketserver.py", line 650, in process_request_thread
self.finish_request(request, client_address)
File "/usr/local/anaconda3/lib/python3.8/socketserver.py", line 360, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/local/anaconda3/lib/python3.8/socketserver.py", line 720, in __init__
self.handle()
File "/usr/local/anaconda3/lib/python3.8/site-packages/django/core/servers/basehttp.py", line 174, in handle
self.handle_one_request()
File "/usr/local/anaconda3/lib/python3.8/site-packages/django/core/servers/basehttp.py", line 182, in handle_one_request
self.raw_requestline = self.rfile.readline(65537)
File "/usr/local/anaconda3/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [Errno 54] Connection reset by peer
Below is my code, I receive some keyword and post request to website. Finally, reply to user
#csrf_exempt
def callback(request):
if request.method == 'POST':
signature = request.META['HTTP_X_LINE_SIGNATURE']
body = request.body.decode('utf-8')
content = "None"
try:
events = parser.parse(body, signature)
except InvalidSignatureError:
return HttpResponseForbidden()
except LineBotApiError:
return HttpResponseBadRequest()
for event in events:
if isinstance(event, MessageEvent):
msg = event.message.text.strip()
if msg.startswith('!'):
msg = msg.replace('!', '')
if msg == 'temp':
content = "HELP"
elif msg.startswith(' '):
content = 'Command not found'
elif ' ' in msg:
info = msg.split(' ')
if len(info) > 2:
content = 'Too many arguments'
else:
ID = info[1]
temp = temperature.TempReport(ID)
content = temp.scrape()
#content = 'Test'
else:
content = 'Unknown command'
line_bot_api.reply_message(
event.reply_token,
TextSendMessage(text=content)
)
print('submit success')
else:
line_bot_api.reply_message(
event.reply_token,
TextSendMessage(text=msg)
)
return HttpResponse()
else:
return HttpResponseBadRequest()
Here is scrape code
def scrape(self):
if not self.postToServer():
return f'ID:{self.ID} submit temperature fail!'
return f'ID:{self.ID} submit temperature successfully
def postToServer(self):
self.session.post(self.url_in, data=self.payload)
sleep(0.1)
resp = self.session.get(self.url_out)
sleep(0.1)
soup = BeautifulSoup(resp.text, features='lxml')
result = soup.find(class_='title-text').text.strip()
return 0 if not 'completed' in result else
The post request work fine and the error happened when I return HttpResponse in callback function. I don't know what is the issue here...

Python BeautifulSoup "weird" errors

Somewhere in the mist i have tangled myself running these code gives me "weird" errors and it seems like i am missing a module but cant seem to get it work even after reading the error messages many times.
Anyone that has a clue on whats wrong here?
Happy new year and thanks in advance!
import requests
from bs4 import BeautifulSoup
import csv
def get_page(url):
response = requests.get(url)
if not response.ok:
print('Server responded:', response.status_code)
else:
soup = BeautifulSoup(response.text, 'lxml')
return soup
def get_detail_data(soup):
try:
product = soup.find('span',{'class':'a-size-large product-title-word-break'}).text
except:
product = ''
try:
price = soup.find('span',{'class':'a-size-medium a-color-price priceBlockBuyingPriceString'}).text.strip()
currency, price = p.split(' ')
except:
currency = ''
price = ''
try:
amount = soup.find('span', class_='a-size-medium a-color-state').find('a').text.strip()
except:
amount = ''
data = {
'product': product,
'price': price,
'currency': currency,
'amount': amount,
}
return data
def get_index_data(soup):
try:
links = soup.find_all('a',class_='a-link-normal a-text-normal')
except:
links = []
urls = [item.get('href') for item in links]
return urls
def write_csv(data, url):
with open('hardware.csv', 'a') as csvfile:
writer = csv.writer(csvfile)
row = [data['title'], data['price'], data['currency'], data['amount'], url]
writer.writerow(row)
def main():
url = 'https://www.amazon.se/s?k=grafikkort&page=1'
products = get_index_data(get_page(url))
for link in products:
data = get_detail_data(get_page(link))
write_csv(data, link)
if __name__ == '__main__':
main()
And the Error messages.
Traceback (most recent call last):
File "scrp.py", line 75, in <module>
main()
File "scrp.py", line 71, in main
data = get_detail_data(get_page(link))
File "scrp.py", line 7, in get_page
response = requests.get(url)
File "/usr/lib/python3/dist-packages/requests/api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "/usr/lib/python3/dist-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 519, in request
prep = self.prepare_request(req)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 452, in prepare_request
p.prepare(
File "/usr/lib/python3/dist-packages/requests/models.py", line 313, in prepare
self.prepare_url(url, params)
File "/usr/lib/python3/dist-packages/requests/models.py", line 387, in prepare_url
raise MissingSchema(error)
requests.exceptions.MissingSchema: Invalid URL '/ASUS-NVIDIA-GeForce-grafikkort-kylning/dp/B07489XSJP?dchild=1': No schema supplied. Perhaps you meant http:///ASUS-NVIDIA-GeForce-grafikkort-kylning/dp/B07489XSJP?dchild=1?
What is happening here is that you are only getting the URL suffixes from your products, as seen for instance with /ASUS-NVIDIA-GeForce-grafikkort-kylning.
A quick solution is to prepend `'https://amazon.se' to all your urls:
def main():
url = 'https://www.amazon.se/s?k=grafikkort&page=1'
products = get_index_data(get_page(url))
for link in products:
data = get_detail_data(get_page('https://www.amazon.se' + link))
write_csv(data, link)

Getting NameError while importing a module

import requests
import json
token= "12882xxxx:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
class T_bot:
def __init__(self):
self.base= "https://api.telegram.org/bot{}/".format(token)
self.sendurl= "https://api.telegram.org/bot{}/sendMessage".format(token)
#self.data= {'chat_id':1067109380, 'text': 'Hi I hope you are doing fine' }
def gett(self, offset=None):
url= self.base+"getUpdates?timeout=100"
if offset:
url = url+"&offset={}".format(offset+1)
a= requests.get(url)
return a.json()
def send(self, msg, chat_id):
url= self.base+"sendMessage?chat_id={}&text={}".format(chat_id,msg)
if msg is not None:
requests.get(url)
when I'm importing the above code (bot.py)as:
from bot import T_bot
update_id= None
def make_reply(msg):
reply= 'okay'
return reply
update_id= None
while True:
updates= T_bot.gett(self, offset= update_id)
updates= updates["result"]
if updates:
for item in updates:
update_id= item['update_id']
try:
message= item['message']['text']
except:
message= None
fromm= item['message']['from']['id']
reply= make_reply(message)
T_bot.send(reply, fromm)
it is throwing a NameError when I'm running the main file above:
Traceback (most recent call last):
File "C:\Users\shivam\Desktop\pypy\server.py", line 13, in <module>
updates= T_bot.gett(self, offset= update_id)
NameError: name 'self' is not defined
I understand that I have to instantiate class first but how can I do it when I'm importing another module. somebody, please explain it!
You will have to instantiate the class first.
Ie.
from bot import T_bot
update_id = None
def make_reply(msg):
reply = 'okay'
return reply
update_id = None
instantiated_class = T_bot()
while True:
updates = instantiated_class.gett(offset=update_id)
updates = updates["result"]
if updates:
for item in updates:
update_id = item['update_id']
try:
message = item['message']['text']
except:
message = None
fromm = item['message']['from']['id']
reply = make_reply(message)
T_bot.send(reply, fromm)

Python - Multiprocessing Pool map returning can't pickle error

I have following code which creates a testrail client and executes testrail's GET_SUITES API call.
I have a function to call the GET_SUITES API and I am passing testrail client & test_rail_project_id as params
I am trying to use multiprocessing to execute over my list of projects to speed up things and I am can't pickle error
My code:
from itertools import product
def get_suites(client, project_id):
try:
path = 'get_suites/{projectid}'.format(projectid=project_id)
test_rail_response = client.send_get(path)
return test_rail_response
except Exception as e:
raise Exception(str(e))
if __name__ == "__main__":
testRailClient = APIClient(TESTRAIL_URL)
pool = Pool(2)
all_project_ids = [100, 200, 300]
data = pool.starmap(get_suites, product([testRailClient], all_project_ids))
Error stack:
Traceback (most recent call last):
File "main.py", line 57, in <module>
data = pool.starmap(testrailapi.get_suites, product([testRailClient], all_project_ids))
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py", line 274, in starmap
return self._map_async(func, iterable, starmapstar, chunksize).get()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py", line 644, in get
raise self._value
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py", line 424, in _handle_tasks
put(task)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/connection.py", line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
TypeError: can't pickle SSLContext objects
Any suggestions please?
Thank you
PS: I am using Python3.6
UPDATE:
As suggested I tried removing the API client as a parameter and it worked but I am getting the same error when I have "get_suites" as a method. Please see my updated code below
class TestRailExecution:
def __init__(self, url, username, password):
self.url = url
self.username = username
self.password = password
self.client = APIClient(self.url)
self.client.user = username
self.client.password = password
def get_suites(self, project_id):
try:
path = 'get_suites/{projectid}'.format(projectid=project_id)
test_rail_response = self.client.send_get(path)
return test_rail_response
except Exception as e:
raise Exception(str(e))
if __name__ == "__main__":
testRailClient = TestRailExecution(TESTRAIL_URL, user, password)
pool = Pool(2)
data = pool.map(get_suites, [100, 200, 300])

Resources