I'm trying to implement concurrent requests to speed up the checking of a list of URL's but it doesn't seem to be working with my code as it's still checking them 1 by 1.
for domain in list:
try:
follow_url = requests.head(f'http://{domain}', allow_redirects=True, timeout=60)
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
executor.submit(follow_url)
with open("alive.txt", "a") as file:
file.write(f'{domain}\n')
except Exception as e:
print(e)
You are not applying it correctly. You are creating parallel processes inside an iteration. Correct way could be like this:
def parallel_req(domain):
try:
follow_url = requests.head(f'http://{domain}', allow_redirects=True, timeout=60)
with open("alive.txt", "a") as file:
file.write(f'{domain}\n')
except requests.exceptions.RequestException as e:
print(e)
with ThreadPoolExecutor() as e:
e.map(parallel_req, domains)
Related
I'm calling two functions at the same time and the functions are triggered, but when the mysql query is made, the code seems to run and stays in an infinite loop, not returning an error.
here is my code:
main.py
import threading
import siparis_kontrolu
import dolar_euro_guncelle
def dd1():
print("a")
siparis_kontrolu.siparis_kontrolu()
def dd2():
print("b")
dolar_euro_guncelle.dolar_euro_guncelle()
t1 = threading.Thread(target=dd1)
t2 = threading.Thread(target=dd2)
t1.start()
t2.start()
siparis_kontrolu.py
import mysql_db
def siparis_kontrolu():
try:
while True:
print("test1")
tum_kullanicilar = mysql_db.mysql_get_all("select * from users")
print("test2")
dolar_euro_guncelle.py
import urllib.request
import json
import mysql_db
import time
import logla
def dolar_euro_guncelle():
while True:
try:
print("f")
data = urllib.request.urlopen(
"https://finans.truncgil.com/today.json")
for line in data:
line = json.loads(line.decode('utf-8'))
USD = round(float(line['USD']["Satış"].replace(",", ".")), 2)
EUR = round(float(line['EUR']["Satış"].replace(",", ".")), 2)
mysql_db.mysql_update("update variables set EUR='" +
str(EUR)+"', USD='"+str(USD)+"' where id='4'")
time.sleep(10)
print("USD/EUR guncellendi.")
except Exception as e:
logla.logla(e, "dolar_euro_guncelle")
print(e)
mysql_db.py
from configparser import ConfigParser
import mysql.connector
config_file = 'config.ini'
config = ConfigParser()
config.read(config_file)
mydb = mysql.connector.connect(host=config['MYSQL']['DB_HOST'],
user=config['MYSQL']['DB_USERNAME'], passwd=config['MYSQL']['DB_PASSWORD'], database=config['MYSQL']['DB_DATABASE'])
mycursor = mydb.cursor(buffered=True, dictionary=True)
def mysql_update(sorgu):
try:
mycursor.execute(sorgu)
mydb.commit()
except Exception as e:
print(e)
def mysql_get_all(sorgu):
try:
mycursor.execute(sorgu)
return mycursor.fetchall()
except Exception as e:
print(e)
When I run main.py these are written to the console:
a
b
f
test1
test2 and USD/EUR guncellendi is not printed, I don't understand exactly what's wrong, when I trigger siparis_kontrolu.py directly without using threading, it works fine
It remains as it appears in the picture and the code does not stop.
But it doesn't do what I want it to do.
It looks like a concurrency issue with the DB connection (assuming your siparis_kontrolu.py doesn't throw syntax errors due to the missing except / finally part of the try statement, a sleep is also missing).
The "SELECT" statements conflict with the "Update" statements.
So it turns out: each thread needs its own database connection!
The easiest way is to open another database connection and create another cursor (maybe another cursor instance is enough) in your mysql_db.py:
from configparser import ConfigParser
import mysql.connector
config_file = 'config.ini'
config = ConfigParser()
config.read(config_file)
mydb = mysql.connector.connect(host=config['MYSQL']['DB_HOST'],
user=config['MYSQL']['DB_USERNAME'], passwd=config['MYSQL']['DB_PASSWORD'], database=config['MYSQL']['DB_DATABASE'])
mycursor = mydb.cursor(buffered=True, dictionary=True)
mydb_upd = mysql.connector.connect(host=config['MYSQL']['DB_HOST'],
user=config['MYSQL']['DB_USERNAME'], passwd=config['MYSQL']['DB_PASSWORD'], database=config['MYSQL']['DB_DATABASE'])
mycursor_upd = mydb_upd.cursor(buffered=True, dictionary=True)
def mysql_update(sorgu):
try:
mycursor_upd.execute(sorgu)
mydb_upd.commit()
except Exception as e:
print(e)
def mysql_get_all(sorgu):
try:
mycursor.execute(sorgu)
return mycursor.fetchall()
except Exception as e:
print(e)
But creating a DbHandler class and use instances of it in the several threads would be much cleaner.
I am trying to implement exception handeling using Pyspark in Databricks, where I need to check the file if it exists in the source location.
df = spark.read.csv.option("inferschema", "true").load("mnt/pnt/abc.csv")
try:
df = open("abc.csv", "rt")
print("File opened")
except FileNotFoundError:
print("File does not exist")
except:
print("Other error")**
I wish to have something like the above code snippet however i am not being able to take the approach. I would request some help would be really thankful
You can't directly Except java.io errors, however you could do something like:
def read_file(path):
try:
dbutils.fs.ls(path)
return spark.read.option("inferschema","true").csv(path)
except Exception as e:
if 'java.io.FileNotFoundException' in str(e):
print('File does not exists')
else:
print('Other error')
read_file('mnt/pnt/abc.csv')
I have a list of IDs that I am passing into a URL within a for loop:
L = [1,2,3]
lst=[]
for i in L:
url = 'URL.Id={}'.format(i)
xml_data1 = requests.get(url).text
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)
I am trying to create a try/catch where regardless of the error, the request library keeps trying to connect to the URL from the ID it left off on from the list (L), how would I do this?
I setup this try/catch from this answer (Correct way to try/except using Python requests module?)
However this forces the system to exit.
try:
for i in L:
url = 'URL.Id={}'.format(i)
xml_data1 = requests.get(url).text
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)
except requests.exceptions.RequestException as e:
print (e)
sys.exit(1)
You can put the try-except block in a loop and only break the loop when the request does not raise an exception:
L = [1,2,3]
lst=[]
for i in L:
url = 'URL.Id={}'.format(i)
while True:
try:
xml_data1 = requests.get(url).text
break
except requests.exceptions.RequestException as e:
print(e)
lst.append(xml_data1)
time.sleep(1)
print(xml_data1)
I Was trying to run this piece of code to generate a directory and a file inside it, the thing is that both the of them won't get created until the program is terminated.
What can i do to create them while the program is still running?
def makeDirectory(dirRoot, path, fileName, fromaddr):
try:
os.mkdir(dirRoot, 0o0755)
except OSError as e:
print(e)
try:
os.mkdir(path, 0o0755)
except OSError as e:
print(e)
path = os.path.abspath(path + '/')
completeName = os.path.join(path, 'foo.txt')
cnt = open(completeName, 'a')
cnt.close()
How can i recover from exception and continue line string from opened file? I'm stuck!
try:
while True:
with open('us.txt') as f:
for user in f:
for tweet in tweepy.Cursor(api.user_timeline, screen_name=user, ).items():
print(tweet.user.screen_name)
csvWriter.writerow(tweet.user.screen_name)
except tweepy.TweepError as e:
print(e.reason)
sys.exit()
If your (admittedly a bit confusing) question is asking how to ignore the exception, but still continue from where you were in the file, you should try and keep your try: and except: blocks as close to the problematic line.
For example, if csvWriter.writerow(tweet.user.screen_name) is the line which fails, you could do:
while True:
with open('us.txt') as f:
for user in f:
for tweet in tweepy.Cursor(api.user_timeline, screen_name=user,).items():
print(tweet.user.screen_name)
try:
csvWriter.writerow(tweet.user.screen_name)
except tweepy.TweepError as e:
print(e)
If instead the error is in the for tweet in tweepy.Cursor(... line, you could do this:
while True:
with open('us.txt') as f:
for user in f:
try:
for tweet in tweepy.Cursor(api.user_timeline, screen_name=user,).items():
print(tweet.user.screen_name)
csvWriter.writerow(tweet.user.screen_name)
except tweepy.TweepError as e:
print(e)
I hope this helps!