Now I create a project. And try create a unauthenticated cache purge scanner program. I tried this but not work
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import requests
with open('new-target.txt', 'rb') as f:
L = f.seek(-2, os.SEEK_END)
while f.read(1) != b'\n':
f.seek(-2, os.SEEK_CUR)
a = f.readline().decode()
filepath = a
with open(filepath) as fp:
line = fp.readline()
cnt = 1
while line:
x = "{}".format(line.strip())
resp = requests.purge(x + "/")
responsee = resp.status_code
if responsee == 200:
print("\033[1;32;40m [Method - PURGE] \033[0m 1;31;40m Vulnerable: \033" + x)
line = fp.readline()
cnt += 1
Gave this error:
Traceback (most recent call last):
File "/root/Masaüstü/Pentesting Tools/PURGE-PUT-Method-Scanner.py", line 23, in
resp = requests.purge(x + "/")
AttributeError: module 'requests' has no attribute 'purge'
Traceback which you got seems to be preety straightforwards. Module requests which you use in this case does not have that purge attribute, which you are trying to call in line 23.
Please refer to methods listed in documentation:
https://docs.python-requests.org/en/latest/api/
Related
This is script to read email and parse body from outlook 365. For me does not work.
I have outlook 365 account without TFA. TFA is not activated
what could be a problem?
#!/usr/bin/python3
from email.message import EmailMessage
import email
import imaplib
import re
import sys
import logging
import base64
import email.parser
import html2text
import requests
import json
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-mpass', '-mailbox_password', dest = 'mailbox_password', help = 'mailbox password.')
args = parser.parse_args()
user = 'someuser#company.com'
mailbox_password ='Email Password' #args.mailbox_password
def get_email_body(body):
if body.is_multipart():
for payload in body.get_payload():
print('To:\t\t', body['To'])
print('From:\t', body['From'])
print('Subject:', body['Subject'])
print('Date:\t',body['Date'])
for part in body.walk():
if (part.get_content_type() == 'text/plain') and (part.get('Content-Disposition') is None):
output = part.get_payload()
else:
print('To:\t\t', body['To'])
print('From:\t', body['From'])
print('Subject:', body['Subject'])
print('Date:\t', body['Date'])
print('Thread-Index:\t', body['Thread-Index'])
text = f"{body.get_payload(decode=True)}"
html = text.replace("b'", "")
h = html2text.HTML2Text()
h.ignore_links = True
output = (h.handle(f'''{html}''').replace("\\r\\n", ""))
output = output.replace("'", "")
# output in one line
#output = output.replace('\n'," ")
output = output.replace('*', "")
return output
def clear_inbox(conn, dest_folder):
output=[]
result = conn.uid('COPY', emailid, dest_folder)
output.append(result)
if result[0] == 'OK':
result = mov, data = conn.uid('STORE',emailid, '+FLAGS', '(\Deleted Items)')
conn.expunge()
conn = imaplib.IMAP4_SSL("outlook.office365.com")
conn.login(user,mailbox_password)
conn.select("Inbox")
try:
resp, items = conn.uid("search",None, 'All')
items = items[0].split()
for emailid in items:
resp, data = conn.uid("fetch",emailid, "(RFC822)")
if resp == 'OK':
email_body = data[0][1].decode('utf-8')
email_message = email.message_from_string(email_body)
subject = email_message["Subject"]
if subject.lower().startswith('Darktrace'.lower()):
output = get_email_body(email_message)
#do some task
# move emails to Processed folder and clear Inbox
clear_inbox(conn, "Processed")
else:
clear_inbox(conn, "backup")
except IndexError:
print("No new email")
conn.close()
conn.logout()
This is error:
Traceback (most recent call last):
File "d:\python\lib\imaplib.py", line 1047, in _command_complete
typ, data = self._get_tagged_response(tag, expect_bye=logout)
File "d:\python\lib\imaplib.py", line 1173, in _get_tagged_response
self._get_response()
File "d:\python\lib\imaplib.py", line 1075, in _get_response
resp = self._get_line()
File "d:\python\lib\imaplib.py", line 1185, in _get_line
raise self.abort('socket error: EOF')
abort: socket error: EOF
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Dev Codes\ReadOutlookEmail.py", line 74, in
resp, data = conn.uid("fetch", emailid, "(RFC822)")
File "d:\python\lib\imaplib.py", line 890, in uid
typ, dat = self._simple_command(name, command, *args)
File "d:\python\lib\imaplib.py", line 1230, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "d:\python\lib\imaplib.py", line 1049, in _command_complete
raise self.abort('command: %s => %s' % (name, val))
abort: command: UID => socket error: EOF
I am a beginner in Python, and trying out making a bot which automatically Tweets anything which is posted on a Subreddit that I have made.
I took help from some of the tutorials online which has the following code
import praw
import json
import requests
import tweepy
import time
access_token = '************************************'
access_token_secret = '************************************'
consumer_key = '************************************'
consumer_secret = '************************************'
def strip_title(title):
if len(title) == 94:
return title
else:
return title[:93] + "..."
def tweet_creator(subreddit_info):
post_dict = {}
post_ids = []
print("[bot] Getting posts from Reddit")
for submission in subreddit_info.get_hot(limit=20):
post_dict[strip_title(submission.title)] = submission.url
post_ids.append(submission.id)
print("[bot] Generating short link using goo.gl")
mini_post_dict = {}
for post in post_dict:
post_title = post
post_link = post_dict[post]
short_link = shorten(post_link)
mini_post_dict[post_title] = short_link
return mini_post_dict, post_ids
def setup_connection_reddit(subreddit):
print("[bot] setting up connection with Reddit")
r = praw.Reddit(' %s' %(subreddit))
subreddit = r.get_subreddit(subreddit)
return subreddit
def shorten(url):
headers = {'content-type': 'application/json'}
payload = {"longUrl": url}
url = "https://www.googleapis.com/urlshortener/v1/url"
r = requests.post(url, data=json.dumps(payload), headers=headers)
link = json.loads(r.text)['id']
return link
def duplicate_check(id):
found = 0
with open('posted_posts.txt', 'r') as file:
for line in file:
if id in line:
found = 1
return found
def add_id_to_file(id):
with open('posted_posts.txt', 'a') as file:
file.write(str(id) + "\n")
def main():
subreddit = setup_connection_reddit('*Name of the subreddit*')
post_dict, post_ids = tweet_creator(subreddit)
tweeter(post_dict, post_ids)
def tweeter(post_dict, post_ids):
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
for post, post_id in zip(post_dict, post_ids):
found = duplicate_check(post_id)
if found == 0:
print("[bot] Posting this link on twitter")
print(post + " " + post_dict[post] + " #Python #reddit #bot")
api.update_status(post+" "+post_dict[post]+" #Python #reddit #bot")
add_id_to_file(post_id)
time.sleep(30)
else:
print("[bot] Already posted")
if __name__ == '__main__':
main()
The code seems fine in PyCharm, however I am getting the following error when I try to run it directly from the folder via Terminal using the rolling code, reddit_bot2.py is my file name:
python3 reddit_bot2.py
When I try to run the code I am getting the following error:
mahesh#Maheshs-MacBook-Air Atoms % python3 reddit_bot2.py
[bot] setting up connection with Reddit
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/configparser.py", line 846, in items
d.update(self._sections[section])
KeyError: '**Name of the subreddit to fetch posts from**'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/mahesh/Python_Bot/Atoms/reddit_bot2.py", line 82, in <module>
main()
File "/Users/mahesh/Python_Bot/Atoms/reddit_bot2.py", line 62, in main
subreddit = setup_connection_reddit('Bot167')
File "/Users/mahesh/Python_Bot/Atoms/reddit_bot2.py", line 36, in setup_connection_reddit
r = praw.Reddit(' %s' %(subreddit))
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/praw/reddit.py", line 227, in __init__
self.config = Config(
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/praw/config.py", line 85, in __init__
self.custom = dict(Config.CONFIG.items(site_name), **settings)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/configparser.py", line 849, in items
raise NoSectionError(section)
configparser.NoSectionError: No section: ' Bot167'
You provided the name of a praw.ini configuration which does not exist.
For help with creating a Reddit instance, visit
https://praw.readthedocs.io/en/latest/code_overview/reddit_instance.html
For help on configuring PRAW, visit
https://praw.readthedocs.io/en/latest/getting_started/configuration.html
Any help in this regards would be highly appreciated.
Thanks :)
I built a script to find the most recent version of Visual Studio Code, download it and install it on an Ubuntu machine with dpkg. I haven't found a decent Python library for doing this and am using subprocess.call() to invoke a Shell command. This certainly may not be the best way to do it, but this is also a learning project.
It successfully downloads the file and places it in my ~/Downloads dir. When I try to invoke the subprocess.call(), it spits back 'OSError: [Errno 9] Bad file descriptor'
I know my command string is correct. I can invoke it just fine from the CLI. But it doesn't work when called through subprocess.
Any advice on doing this more efficiently is welcome.
"""
Python 3 script
Downloads the latest .deb package for installing VSCode, and installs it
"""
import os # used to direct where to save downloaded file
import subprocess # used to derive filepath of CLI arg
import requests # py3 only
import platform # used to detect the OS
from urllib.request import urlopen, ContentTooShortError, urlretrieve # py3 version of 'import urllib2'
HOME = os.path.expanduser('~')
filePath = HOME + "/Downloads"
fileName = 'vs_code_most_recent_amd64.deb'
outputName = os.path.join(filePath, fileName)
alreadyDownloaded = False
# used in subprocess calls to suppress stdout or stderr
pipeToDevNull = open(os.devnull, 'w')
def IsDownloadable(url):
"""
Check of the link passed in is a downloadable file. Used to shortcut the
processing so that it doesn't attempt to download a URL that isn't
downloadable. Returns True or False.
"""
h = requests.head(url, allow_redirects=True)
header = h.headers
contentType = header.get('content-type')
if 'text' in contentType.lower():
return False
if 'html' in contentType.lower():
return False
return True
def DownloadVSCodePkg(url):
"""
Downloads the file at the specified URL, save it as the above-defined filename
"""
u = urlopen(url)
f = open(outputName, 'wb')
meta = u.info()
fileSize = int(meta.get_all("Content-Length")[0])
fileSizeDL = 0
#blockSize = 8192
blockSize = 16384
while True:
buffer = u.read(blockSize)
if not buffer:
break
fileSizeDL += len(buffer)
f.write(buffer)
status = r"%10d Bytes [%3.2f%%]" % (fileSizeDL, fileSizeDL * 100. / fileSize)
status = status + chr(8)*(len(status)+1)
print("Downloading: {0}".format(status), end="\r", flush=True)
print("Downloading: {0}".format(status))
print("Downloaded: {0}".format(fileName))
f.close()
del f
def CheckDownloadSuccess():
"""
returns bool value if the file we want is in the dir specified
"""
try:
subprocess.check_call("ls " + outputName, stdout=pipeToDevNull, stderr=pipeToDevNull, shell=True)
return True
except subprocess.CalledProcessError:
return False
def UnpackAndInstall():
"""
Invokes dpkg from the linux shell and installs VSCode.
"""
#Detect OS
linuxDistro = platform.linux_distribution()
OSType = linuxDistro[0]
if OSType == 'Ubuntu':
from apt.debfile import DebPackage
pkg = DebPackage(outputName)
command = 'sudo dpkg -i ' + outputName
#The thing that attempts to unpack:
try:
subprocess.check_call(command, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError:
print("Install Failed.")
def main():
url = 'https://go.microsoft.com/fwlink/?LinkID=760868'
alreadyDownloaded = CheckDownloadSuccess()
if alreadyDownloaded is False:
if IsDownloadable(url):
DownloadVSCodePkg(url)
# check if the download succeeded, if file doesn't already exist.
if CheckDownloadSuccess():
print("Download Successful!\nFile location => " + outputName)
else:
print("Download Failed...")
else:
print('Link broken: need to update the package resource link.')
else:
print("File already exists.")
UnpackAndInstall()
if __name__ == "__main__":
main()
Here is the traceback and the error from the CLI:
$ python3 setupVSCode.py
Traceback (most recent call last):
File "setupVSCode.py", line 192, in <module>
main()
File "setupVSCode.py", line 189, in main
UnpackAndInstall()
File "setupVSCode.py", line 95, in UnpackAndInstall
subprocess.call(command, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT, shell=True)
File "/usr/lib/python3.6/subprocess.py", line 267, in call
with Popen(*popenargs, **kwargs) as p:
File "/usr/lib/python3.6/subprocess.py", line 709, in __init__
restore_signals, start_new_session)
File "/usr/lib/python3.6/subprocess.py", line 1344, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
OSError: [Errno 9] Bad file descriptor
os.path.expanduser('~') is going to return something like: 'C:\\Users\\user.name' which you're appending '/Downloads', resulting in a bad path like: 'C:\\Users\\user.name/Downloads\\'
Instead of:
filePath = HOME + "/Downloads"
Do:
filePath = HOME + "\Downloads"
Or preferably:
filePath = os.path.join(HOME, 'Downloads')
After speaking with #Steve, I tried removing the output redirectors on the subprocess.call().
The advice on removing all slashes in path construction and instead using "os.path.join()" has been implemented and will be followed as a best practice from here on.
Since the command as constructed worked fine from the CLI, it was a matter of thinking about what the subprocess.call() did that was different. It redirected output. With that removed things work fine
It now looks like so:
HOME = os.path.expanduser('~')
filePath = os.path.join(HOME, "Downloads")
fileName = 'vs_code_most_recent_amd64.deb'
outputName = os.path.join(filePath, fileName)
alreadyDownloaded = False
...
command = 'sudo dpkg -i ' + outputName
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("Install Failed.")
I am on macOS 10.14.2 using "homebrewed" python3.7.2, scrapy 1.5.1 and twisted 18.9.0 as a python novice with the following script to download old newspaper archived on a website:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# A scrapy script to download issues of the Gaceta (1843-1961)
import errno
import json
import os
from datetime import datetime
import scrapy
from scrapy import FormRequest, Request
os.chdir("/Volumes/backup/Archives/Gaceta_Nicaragua_1843-1961") # directory path
print((os.getcwd()))
# date range, format DD/MM/YYYY
start = '01/01/1843' # 01/01/1843
end = '31/12/1860' # 31/12/1961
date_format = '%d/%m/%Y'
start = datetime.strptime(start, date_format)
end = datetime.strptime(end, date_format)
class AsambleaSpider(scrapy.Spider):
name = 'asamblea'
allowed_domains = ['asamblea.gob.ni']
start_urls = ['http://digesto.asamblea.gob.ni/consultas/coleccion/']
papers = {
"Diario Oficial": "28",
}
def parse(self, response):
for key, value in list(self.papers.items()):
yield FormRequest(url='http://digesto.asamblea.gob.ni/consultas/util/ws/proxy.php',
headers= {
'X-Requested-With': 'XMLHttpRequest'
}, formdata= {
'hddQueryType': 'initgetRdds',
'cole': value
}
, meta={'paper': key},
callback=self.parse_rdds
)
pass
def parse_rdds(self, response):
data = json.loads(response.body_as_unicode())
for r in data["rdds"]:
if not r['fecPublica']:
continue
r_date = datetime.strptime(r['fecPublica'], date_format)
if start <= r_date <= end:
r['paper'] = response.meta['paper']
rddid = r['rddid']
yield Request("http://digesto.asamblea.gob.ni/consultas/util/pdf.php?type=rdd&rdd=" + rddid,
callback=self.download_pdf, meta=r)
def download_pdf(self, response):
filename = "{paper}/{anio}/".format(**response.meta) + "{titulo}-{fecPublica}.pdf".format(**response.meta).replace("/", "_")
if not os.path.exists(os.path.dirname(filename)):
try:
os.makedirs(os.path.dirname(filename))
except OSError as exc: # guard against race condition
if exc.errno != errno.EEXIST:
raise
with open(filename, 'wb') as f:
f.write(response.body)
It worked perfectly fine (although slow), however, I have two persisting issues with the script.
Firstly, I get the following error since the update:
2019-01-07 11:53:34 [scrapy.core.scraper] ERROR: Spider error processing <POST http://digesto.asamblea.gob.ni/consultas/util/ws/proxy.php> (referer: http://digesto.asamblea.gob.ni/consultas/coleccion/)
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/scrapy/utils/defer.py", line 102, in iter_errback
yield next(it)
File "/usr/local/lib/python3.7/site-packages/scrapy/spidermiddlewares/offsite.py", line 30, in process_spider_output
for x in result:
File "/usr/local/lib/python3.7/site-packages/scrapy/spidermiddlewares/referer.py", line 339, in <genexpr>
return (_set_referer(r) for r in result or ())
File "/usr/local/lib/python3.7/site-packages/scrapy/spidermiddlewares/urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
File "/usr/local/lib/python3.7/site-packages/scrapy/spidermiddlewares/depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
File "gaceta_downloader.py", line 58, in parse_rdds
if not r['fecPublica']:
KeyError: 'fecPublica'
Secondly, once the script runs again (as it did some days ago before updating python and packages) I ran into an issue where the script would sometimes complain that UnicodeEncodeError: ‘ascii’ codec can’t encode character u’\xb0’ in position 27: ordinal not in range(128) which I guess let sometimes to zero byte files. Do you see the encoding error in the source code? Is this related to the above problem?
I have some code that works when I run it on a Windows machine, but when it runs in Ubuntu on a google ComputeEngine VM I get the following error.
Traceback (most recent call last): File "firehose_get.py", line 43,
in
print(json.dumps(json.loads(line),indent=2)) File "/home/stuartkirkup/anaconda3/lib/python3.5/json/init.py", line
312, in loads
s.class.name)) TypeError: the JSON object must be str, not 'bytes'
It's exactly the same code that runs fine on Windows. I've done quite a bit of reading and it looks like an encoding issue - and as you'll see from some of the commented out sections in my code I've tried some ways to change the encoding but without joy. I've tried various things but can't work out how to debug it ... I'm fairly new to Python
I'm using Anaconda which some further reading says it has an ill advised setdefaultencoding hack built in.
Here is the stream header showing it's chunked data, which I believe is why it's bytes
{'Transfer-Encoding': 'chunked', 'Date': 'Thu, 17 Aug 2017 16:53:35 GMT', 'Content-Type': 'application/json', 'x-se
rver': 'db220', 'Content-Encoding': 'gzip'}
Code file - firehose_requests.py (with api keys infor replaced by ####)
import requests
MAX_REDIRECTS = 1000
def get(url, **kwargs):
kwargs.setdefault('allow_redirects', False)
for i in range(0, MAX_REDIRECTS):
response = requests.get(url, **kwargs)
#response.encoding = 'utf-8'
print ("test")
print (response.headers)
if response.status_code == requests.codes.moved or \
response.status_code == requests.codes.found:
if 'Location' in response.headers:
url = response.headers['Location']
content_type_header = response.headers.get('content_type')
print (content_type_header)
continue
else:
print ("Error when reading the Location field from HTTP headers")
return response
Code file - firehose_get.py
import json
import requests
from time import sleep
import argparse
#import ConfigParser
import firehose_requests
from requests.auth import HTTPBasicAuth
# Make it work for Python 2+3 and with Unicode
import io
try:
to_unicode = unicode
except NameError:
to_unicode = str
#request a token from Adobe
request_access_token = requests.post('https://api.omniture.com/token', data={'grant_type':'client_credentials'}, auth=HTTPBasicAuth('##############-livestream-poc','488##############1')).json()
#print(request_access_token)
#grab the token from the JSON returned
access_token = request_access_token["access_token"]
print(access_token)
url = 'https://livestream.adobe.net/api/1/stream/eecoukvanilla-##############'
sleep_sec=0
rec_count=10
bearer = "Bearer " + access_token
headers = {"Authorization": bearer,"accept-encoding":"gzip,deflate"}
r = firehose_requests.get(url, stream=True, headers=headers)
#open empty file
with open('output_file2.txt', 'w') as outfile:
print('', file=outfile)
#Read the Stream
if r.status_code == requests.codes.ok:
count = 0
for line in r.iter_lines():
if line:
#write to screen
print ("\r\n")
print(json.dumps(json.loads(line),indent=2))
#append data to file
with open('output_file2.txt', 'a') as outfile:
print("\r\n", file=outfile)
print(json.dumps(json.loads(line),ensure_ascii = False),file=outfile)
#with io.open('output_file2.txt', 'w', encoding='utf8') as outfile:
# str_ = json.dumps(json.loads(line),
# indent=4, sort_keys=True,
# separators=(',', ': '), ensure_ascii=False)
# outfile.write(to_unicode(str_))
#Break the loop if there are is a -n argument
if rec_count is not None:
count = count + 1
if count >= rec_count:
break
#How long to wait between writes
if sleep_sec is not None :
sleep(sleep_sec)
else:
print ("There was a problem with the Request")
print ("Returned Status Code: " + str(r.status_code))
Thanks
OK I worked it out. I found a lot of people also getting this error but no solutions posted, so this is how I did it
parse and decode the JSON like this
json_parsed = json.loads(line.decode("utf-8"))
Full code:
import json
import requests
from time import sleep
import argparse
#import ConfigParser
import firehose_requests
from requests.auth import HTTPBasicAuth
# Make it work for Python 2+3 and with Unicode
import io
try:
to_unicode = unicode
except NameError:
to_unicode = str
#request a token from Adobe
request_access_token = requests.post('https://api.omniture.com/token', data={'grant_type':'client_credentials'}, auth=HTTPBasicAuth('##########-livestream-poc','488################1')).json()
#print(request_access_token)
#grab the token from the JSON returned
access_token = request_access_token["access_token"]
print(access_token)
url = 'https://livestream.adobe.net/api/1/stream/##################'
sleep_sec=0
rec_count=10
bearer = "Bearer " + access_token
headers = {"Authorization": bearer,"accept-encoding":"gzip,deflate"}
r = firehose_requests.get(url, stream=True, headers=headers, )
#open empty file
with open('output_file.txt', 'w') as outfile:
print('', file=outfile)
#Read the Stream
if r.status_code == requests.codes.ok:
count = 0
for line in r.iter_lines():
if line:
#parse and decode the JSON
json_parsed = json.loads(line.decode("utf-8"))
#write to screen
#print (str(json_parsed))
#append data to file
with open('output_file.txt', 'a') as outfile:
#write to file
print(json_parsed,file=outfile)
#Break the loop if there are is a -n argument
if rec_count is not None:
count = count + 1
if count >= rec_count:
break
#How long to wait between writes
if sleep_sec is not None :
sleep(sleep_sec)
else:
print ("There was a problem with the Request")
print ("Returned Status Code: " + str(r.status_code))