I made a script :
import os , time
from pathlib import Path
from pushover import Client
client = Client("MY CLIENT KEY", api_token="MY API KEY")
###Run aquatone
###Create a directory named site if not exists
###copy urls.txt in site directory
###Check for urls in urls.txt
domains = open('/root/programming/aquarecon/domains.txt', 'r')
hue = domains.readlines()
for site in hue:
#run_aquatone = os.system('aquatone-discover -d ' + site.strip('\n') + ' -t 30')
#run_aquatone_scan = os.system('aquatone-scan -d ' + site.strip('\n') + ' -t 30 -p huge')
current_directory = os.getcwd()
url_txt_path = '/root/aquatone/' + site.strip('\n') + '/urls.txt'
#print(url_txt_path)
site_directory = Path("/root/programming/aquarecon/"+ site.strip('\n'))
time.sleep(1)
if site_directory.is_dir() == False:
print('Site Have Not Been Monitored Yet')
os.system('mkdir '+ '' + '/root/programming/aquarecon/'+site.strip('\n'))
copy_url_txt = os.system('cp '+ url_txt_path + ' ' + site.strip('\n') )
else:
print('Site Exists time to use flashback')
open_new_urls = open('/root/aquatone/' + site.strip('\n') + '/urls.txt','r')
open_old_urls = open('/root/programming/aquarecon/' +site.strip('\n') +'/urls.txt' , 'r')
readlines_old = open_old_urls.readlines()
readlines_new = open_new_urls.readlines()
client.send_message('New Subdomain Found ', title="CRON")
for i in readlines_new:
if i not in readlines_old:
client.send_message('New Subdomain Found ' + i, title="AquaRecon")
copy_url_txt = os.system('cp '+ url_txt_path + ' ' + site.strip('\n'))
and Even i set the cronjob as :
10 * * * * /usr/bin/python3.5 /root/programming/aquarecon/aquarecon.py
But the script doesn't run anytime .
For a quick info:
Aquatone is a tool to enumerate subdomains :)
Any help would be appreciated.
Thanks
I believe cron is based off of root, so when you call the pathlib, it is cron calling it, defaulting it to root.
Related
I have successfully implemented an API that generates a unique URL to grab data from a database and downloads it into a csv. I am now attempting to automate this API so that it can generate the unique URL every hour and then append the csv file with the new data. I have no idea where to begin to automate this but the working API is pasted below so any help would be truly appreciated. Thank you.
import os
import sys
from datetime import datetime
from os.path import expanduser
import urllib.request
def main():
# API parameters
options = {}
options["url"] = "https://airnowapi.org/aq/data/"
options["start_date"] = "2020-01-01"
options["start_hour_utc"] = "01"
options["end_date"] = "2020-01-01"
options["end_hour_utc"] = "05"
options["parameters"] = "pm25"
options["bbox"] = "-76,38,-72,42"
options["data_type"] = "b"
options["format"] = "text/csv"
options["ext"] = "csv"
options["api_key"] = "" #NotIncludedforProtectionOfUniqueAPIkey
# API request URL
REQUEST_URL = options["url"] \
+ "?startdate=" + options["start_date"] \
+ "t" + options["start_hour_utc"] \
+ "&enddate=" + options["end_date"] \
+ "t" + options["end_hour_utc"] \
+ "¶meters=" + options["parameters"] \
+ "&bbox=" + options["bbox"] \
+ "&datatype=" + options["data_type"] \
+ "&format=" + options["format"] \
+ "&api_key=" + options["api_key"]
try:
# Request AirNowAPI data
print("Requesting AirNowAPI data...")
print(REQUEST_URL)
# User's home directory.
home_dir = expanduser("E:\SPRING2021\AIRNOWAPI\AIRNOWFILES")
download_file_name = "AirNowAPI" + datetime.now().strftime("_%Y%M%d%H%M%S." + options["ext"])
download_file = os.path.join(home_dir, download_file_name)
# Perform the AirNow API data request
api_data = urllib.request.URLopener()
api_data.retrieve(REQUEST_URL, download_file)
# Download complete
print ("Download URL: %s" % REQUEST_URL)
print("Download File: %s" % download_file)
except Exception as e:
print("Unable perform AirNowAPI request. %s" % e)
sys.exit(1)
if __name__ == "__main__":
main()
I find most of your code is well documented. There are many ways to automated your task. Here are the steps I would recommend you to do.
Create a Config file.
Try to separate your code from config(All the options data). You can even pickle it.
Make your code command line executable like python main.py config.yml, where you can pass config.file.
Checkpoint: Here your code should be one/multiple file and config is in another file.
Use a cronjob or any scheduler to trigger & step 3.
Shared Variables/Data?: If you have variables that need to be passed from first execution to another then you can use a static file, where you dump this data and using it for next interation
so as you can see im trying to create a small backup script for my self, to select needed files and back them up.
import shutil
import datetime
import os
import time
def backup():
# set the update interval
while True:
backup_interval = input("Please enter the backup interval in seconds: ") # 300
try:
valid_time = int(backup_interval) // 60
print("Backup time set to:", valid_time, "minutes!")
break
except ValueError:
print("This time is not valid, please enter a correct time in seconds: ")
print(">>> 60 seconds = 1 minute, 3600 seconds = 60 minutes.")
backup_file = input(r"Please enter the path for the file to backup: ") # D:\Python\BackupDB\test.db"
dest_dir = input(r"Please enter the destination path: ") # D:\Python\BackupDB\
folder_name = input(r"Please name your backup folder: ") # BD_Backup
now = str(datetime.datetime.now())[:19]
now = now.replace(":", "_")
# backup_file = backup_file.replace(backup_file, backup_file + str(now) + ".db")
# thats why I got the FileNotFoundError
final_destination = os.path.join(dest_dir, folder_name)
if not os.path.exists(final_destination):
os.makedirs(final_destination)
print("hello world")
shutil.copy(backup_file, final_destination)
the first question is, how do i replace the name after i copied the file into the destination folder to get something like that test.db -> test_2020-02-23 08_36_22.db
like here :
source_dir = r"D:\Python\BackupDB\test.db"
destination_dir = r"D:\Python\BackupDB\BD_Backup\test_" + str(now) + ".db"
shutil.copy(source_dir, destination_dir)
output :
test_2020-02-23 08_36_22.db
what im doing wrong here?
and how to copy the file 5 times and after a while (backup_interval) delete the first one and move the last 4 up and create a new one so I have in total 5 copies of that file?
I have modified your code as you need,
backup_file = input(r"Please enter the path for the file to backup: ") # D:\Python\BackupDB\test.db"
dest_dir = input(r"Please enter the destination path: ") # D:\Python\BackupDB\
folder_name = input(r"Please name your backup folder: ") # BD_Backup
old_file_name=backup_file.split("/")[-1]
now = str(datetime.datetime.now())[:19]
now = now.replace(":", "_")
new_file_name = old_file_name.split(".")[0]+"_" + str(now) + ".db"
final_destination = os.path.join(dest_dir, folder_name)
if not os.path.exists(final_destination):
os.mkdir(final_destination)
new_file="/"+new_file_name
shutil.copy(backup_file, final_destination)
os.rename(final_destination+'/'+old_file_name,final_destination+new_file)
I did like , after copy the file, i just rename it
I've ran into a similar problem before and the reason was that the creation of the directory hadn't completed yet before trying to access it. A simple sleep before the copy should be able to confirm this.
I use a python script to webscrape for "Show Notes" and an mp3. When I encounter a page that has no show notes, this means the show was a Best Of, so I want to skip the download of the notes and mp3. I am not sure where the best place to insert the test would be. The snippet is as follows:
for show_html in showpage_htmls:
try:
p_html = s.get(show_html)
p_soup = BeautifulSoup(p_html.content, 'html.parser')
# set title for SHOW NOTES
title = ''
title = p_soup.title.contents[0]
# get SHOW NOTES chunk and remove unwanted characters (original mp3notes not changed)
mp3notes = ''
mp3notes = p_soup.find('div', {'class': 'module-text'}).find('div')
mp3notes = str(title) + str('\n') + str(mp3notes).replace('<div>','').replace('<h2>','').replace('</h2>','\n').replace('<p>','').replace('<br/>\n','\n').replace('<br/>','\n').replace('</p>','').replace('</div>','').replace('\u2032','')
# FIXME need to skip d/l if no notes
# set basename, mp3named and mp3showtxt
mp3basename = '{0}{1}{2}'.format(show_html.split('/')[3],show_html.split('/')[4],show_html.split('/')[5])
if (os.name == 'nt'):
mp3showtxt = mp3dir + '\\' + mp3basename + '.txt'
mp3named = mp3dir + '\\' + mp3basename + '.mp3'
else:
mp3showtxt = mp3dir + '/' + mp3basename + '.txt'
mp3named = mp3dir + '/' + mp3basename + '.mp3'
# save show notes to local
with open(mp3showtxt, 'w') as f:
try:
f.write(mp3notes)
print("Show notes " + mp3basename + " saved.")
except UnicodeEncodeError:
print("A charmap encoding ERROR occurred.")
print("Show notes for " + mp3basename + ".mp3 FAILED, but continuing")
finally:
f.close()
# FIXME need eyed3 to set mp3 tags since B&T are lazy
# get Full Show mp3 link
mp3url = p_soup.find('a', href = True, string = 'Full Show').get('href')
# get and save mp3
r = requests.get(mp3url)
with open(mp3named, 'wb') as f:
f.write(r.content)
print("Downloaded " + mp3basename + ".mp3.")
except AttributeError:
print(show_html + " did not exist as named.")
I would think an
if not (len(mp3notes) >= 50)
would work; just not sure where to put it or there is better way (more Pythonic).
Ideally, if the mp3notes are less than expected, no notes or mp3 for that show_html would be saved, and the script would start at the next show_html page.
Since I am new to Python, feel free to offer suggestions to making this more Pythonic as well; I am here to learn! Thanks.
Hi I have recently made this script to rename files I scan for work with a prefix and a date. It works pretty well however it would be great if it could make a directory in the current directory with the same name as the first file then move all the scanned files there. E.g. First file is renamed to 'Scanned As At 22-03-2012 0' then a directory called 'Scanned As At 22-03-2012 0' (Path being M:\Claire\Scanned As At 22-03-2012 0) is made and that file is placed in there.
I'm having a hard time figuring out the best way to do this. Thanks in advance!
import os
import datetime
#target = input( 'Enter full directory path: ')
#prefix = input( 'Enter prefix: ')
target = 'M://Claire//'
prefix = 'Scanned As At '
os.chdir(target)
allfiles = os.listdir(target)
count = 0
for filename in allfiles:
t = os.path.getmtime(filename)
v = datetime.datetime.fromtimestamp(t)
x = v.strftime( ' %d-%m-%Y')
os.rename(filename, prefix + x + " "+str(count) +".pdf")
count +=1
Not quite clear about your requirement. If not rename the file, only put it under the directory, then you can use the following codes (only the for-loop of your example):
for filename in allfiles:
if not os.isfile(filename): continue
t = os.path.getmtime(filename)
v = datetime.datetime.fromtimestamp(t)
x = v.strftime( ' %d-%m-%Y')
dirname = prefix + x + " " + str(count)
target = os.path.join(dirname, filename)
os.renames(filename, target)
count +=1
You can check help(os.renames).
Setup is following:
Apache with WSGI successfully set up and running on bare bones application
import sys
# Path of the directory where scripts directory is located.
sys.path.insert(0, 'C:\\Users\\xxx\\Documents\\Programming\\www\\scripts')
from Blog import Blog #Blog.py is a file in the scripts directory
def application(environ, start_response):
status = '200 OK'
output = ''
b = Blog()
for key in environ.keys():
output = output + str(key) + ' : ' + str(environ[key]) + '<br/>'
output = "Test: " + b.title + "<br/>" + b.text + "<br/>" + output
output = b.get_blog() + "<br/>" + output
response_headers = [('Content-type', 'text/html'),
('Content-Length', str(len(output)))]
start_response(status, response_headers)
return [output]
the relevant parts of Blog.py looks like this
class Blog:
#blogid is a str
def get_blog(self):
conn = sqlite3.connect(self.database)
c = conn.cursor()
# get the blog
c.execute('select * from blogs')
results = []
for row in c:
results.append(row)
c.close()
return results
Apache error log gives me:
line 21, in application
output = b.get_blog() + "<br/>" + output
AttributeError: Blog instance has no attribute 'get_blog'
Changing the b.get_blog to str(dir(b)) gives me:
['doc', 'init', 'module', 'get_data', 'text', 'title'] which is an old version of the Blog class, one that I included in the wsgi file a while ago. I can't find where it's being cached or then why it's not being over written by the Blog import because if I change the import to just import Blog and the instantiation to Blog.Blog() it still gives the same dir contents.
Imported modules have an '__file__' attribute. Print that out, it will tell you where the file is that it loaded for the module. Then you can work out the conflict.
Maybe a version elsewhere on module search path or could be an old .pyc file.