An instance of the Rabbit class is created in a for loop and given a datetime object as init variable
but all instances get the same datetime value.
I expected a time difference for each instance; what am I missing in understanding here?
from datetime import datetime
class Rabbit_birth:
num_eyes = 2 # <- Class variable
num_legs = 4
num_ears = 2
num_tails = 1
num_of_rabbits = 0
def __init__(self, name = 'Rabbit',
fur_colour = 'unknown',
eye_colour = 'unknown',
sex = 'unsexed',
breed = 'unknown',
dob = datetime.now(),
tob = datetime.now()
):
self.name = name # <- Instance variable
self.fur_colour = fur_colour
self.eye_colour = eye_colour
self.sex = sex
self.breed = breed
self.dob = str(dob.strftime("%x"))
self.tob = str(tob.strftime("%X"))
Rabbit_birth.num_of_rabbits += 1
def display_rabbit(self):
dob_time = self.dob+" at "+self.tob
return ("{0.name} is a {0.sex} rabbit"
" and has {0.fur_colour} fur and "
"{0.eye_colour} eyes, {0.name} is "
"of the {0.breed} breed of rabbits.\n"
"{0.name}'s DOB is ").format(self) + dob_time
################################################
print()
print('Number of rabbits is ',Rabbit_birth.num_of_rabbits)
rabbits=[]
def create_rabbit():
lr=[]
rabbit = ('name', 'fur colour', 'eye colour', 'sex', 'breed')
for item in rabbit:
r = input(item+' >')
lr.append(r)
r0,r1,r2,r3,r4=lr
name = Rabbit_birth(r0,r1,r2,r3,r4)
return name
for i in range(3):
name=create_rabbit()
rabbits.append(name)
print(rabbits[i].display_rabbit())
I see the mistake I have made, it's a matter of passing the datetime object in the wrong scope.
Here is the relevant corrected code:
## <- class experiment
from datetime import datetime
class Rabbit_birth:
num_eyes = 2 # <- Class variable
num_legs = 4
num_ears = 2
num_tails = 1
num_of_rabbits = 0
def __init__(self, name = 'Rabbit',
fur_colour = 'unknown',
eye_colour = 'unknown',
sex = 'unsexed',
breed = 'unknown',
dob = None, # This line changed
tob = None # This line changed
):
self.name = name # <- Instance variable
self.fur_colour = fur_colour
self.eye_colour = eye_colour
self.sex = sex
self.breed = breed
self.dob = datetime.now() # This line changed
self.tob = datetime.now() # This line changed
Rabbit_birth.num_of_rabbits += 1
def display_rabbit(self):
# This line below changed
dob_time = str(self.dob.strftime("%x"))+" at "+str(self.tob.strftime("%X"))
return ("{0.name} is a {0.sex} rabbit"
" and has {0.fur_colour} fur and "
"{0.eye_colour} eyes, {0.name} is "
"of the {0.breed} breed of rabbits.\n"
"{0.name}'s DOB is ").format(self) + dob_time
################################################
Related
My electric vehicle charging algorithm aims at returning the shortest path length by a charging drone. The drone will meet the electric vehicle at a rendezvous point and charge the vehicle. The vehicle selection is done based on the charging urgency of the vehicles. The total path length is calculated when all vehicles are charged based on their urgency. The pseudocode is:
EDF(List_Req)
SunchrgEV = List_Req
Pathlen = 0
Nchgreq = 0
while(SunchrgEV = 0):
U obtains the updated location of all ei element of SunchrgEV via message exchange
S'gcs = set of GCS element of Sgcs
Gr = min {Eucleadian distance between Loc(ei) and Loc(Gj)} //The closest GCS
ex = min [{Eucleadian distance between Loc(ei) and Loc(Gr)}/ResidDist(ei)] // most urgent EV
if SoC(U)<RqB2D(ex) // then U(the drone) itself needs charge and it reports its termination to the server
// the server dispatches another U(drone) to charge the remaining EVs (ei)
Gr = min {Eucleadian distance between Loc(U) and Loc(Gj)} //The closest GCS to U where U goes to full charge
end if
t = tcurrent
// Finding rendezvous point where ex and U meets
RdvLoc = FindRdvLoc(ex, Loc(ex), Loc(U),t)
if RdvLoc is out of service area of U then
Report2server(ex,'Outofservicearea')
continue
end if
Pathlen += Dist2(Loc(U),RdvLoc)
U sends RdvLoc to ex and flies to RdvLoc
if U reaches ex in ChgRng(U) then
{Pathlen += Charge(ex, RdvLoc)
Nchgreq ++
}
else
Report2server(ex, 'Outofservicearea')
endif
SunchrgEV -= ex
do
Update (Loc(U))
Gr = min {Eucleadian distance between Loc(U) and Loc(Gj)} //U returns to Gr
Pathlen += Dist2(Loc(U), Loc(Gr))
return Pathlen and Nchgreq
//Compute the expected location where U meets e on Map(e)
def FindRdvLoc(e,Le,Lu,t):
Compute X = (x,y) on Rte(e,t) so that {Dist(e,X)/Speed(e,t)} = {Dist(U, X-ChgRng(U))/Speed(U)}
return X
def Charge(ei, RdvLoc):
U starts to charge ei and follows Rte(ei,t)
eLoc = FindLoc(Map(ei), Loc(ei), ChgTime(ei), Speed(ei,t))
return DistM(Map(ei), RdvLoc, eLoc)
The code that I have written so far gives the same output regardless of the input. The code is:
import math
import sys
from typing import List, Any
class Location:
x = 0
y = 0
def __init__(self, x, y):
self.x = x
self.y = y
class Result:
path_len: int = 0
n_charge_request: int = 0
def __init__(self, path_len, n_charge_request):
self.path_len = path_len
self.n_charge_request = n_charge_request
def to_string(self):
return "path len : " + str(self.path_len) + ", n_charge_request : " + str(self.n_charge_request)
def print(self):
print(self.to_string())
#SGCs
class ChargingStation:
def __init__(self, location: Location, fuel):
self.location = location
self.fuel = fuel
#EVs(ex)
class Vehicle:
location = Location(0, 0)
fuel = 1
isNeedEmergencyFuel = False
per_fuel_distance_travel = 2
def __init__(self, id, location, fuel):
self.id = id
self.location = location
self.fuel = fuel
# Resid (U)
def residual_distance(self):
return self.fuel * self.per_fuel_distance_travel # assuming each watt or kw fuel will yield to 2 killos or milies
# RqB2D
def requested_amount_of_charge(self, nearest_charge_station_location: Location) -> int:
distance = get_distance(self.location, nearest_charge_station_location)
cover = self.fuel * self.per_fuel_distance_travel
diff = math.fabs(distance - cover)
if diff > 0:
needed_fuel = diff / self.per_fuel_distance_travel + 2
return needed_fuel
return 0
# U(i)
class Drone:
location = Location(0, 0)
fuel = 0
isFlying = False
isSendForEvCharge = False
per_fuel_distance_travel = 20
serving_radius = 15
G = [
ChargingStation(Location(20, 10), 50),
ChargingStation(Location(50, 80), 40),
ChargingStation(Location(30, 30), 60)
]
def __init__(self, location, fuel):
self.location = location
self.fuel = fuel
# Resid (U)
def residual_distance(self):
return self.fuel * self.per_fuel_distance_travel # assuming each unit of fuel will yield to 2 kilos or miles
def is_out_of_service_zone(self, vehicle_location: Location): # ----->
distance = get_distance(self.location, vehicle_location)
return distance > self.serving_radius
#staticmethod
def get_distance(from_location, to_location):
x_dis = to_location.x - from_location.x
y_dis = to_location.y - from_location.y
x_dis_sqr = math.pow(x_dis, 2)
y_dis_sqr = math.pow(y_dis, 2)
final_dis_sum = x_dis_sqr + y_dis_sqr
final = math.sqrt(final_dis_sum)
return final
class EDFGenerator:
min_fuel = 50
charging_stations_for_drones = [ChargingStation(Location(2, 10), 80), ChargingStation(Location(2, 10), 50),
ChargingStation(Location(2, 10), 100)]
list_waiting_drones = [Drone(Location(5, 10), 50), Drone(Location(2, 10), 50), Drone(Location(2, 10), 50)]
list_sent_drones = []
list_charging_drones = []
def __init__(self):
pass
def rdv_loc(self, ei: Vehicle, drone_location: Location, t: int) -> Location | None:
needed_fuel = ei.requested_amount_of_charge(drone_location)
nearest_charge_station = self.get_nearest_charge_station(ei.location, self.charging_stations_for_drones)
needed_fuel_at_nearest_station = needed_fuel / ei.per_fuel_distance_travel
if nearest_charge_station.fuel < needed_fuel_at_nearest_station:
return None
else:
return nearest_charge_station.location
#staticmethod
def get_nearest_charge_station(from_location: Location, list_of_stations: List[ChargingStation]) -> ChargingStation:
nearest = list_of_stations[0]
min_distance = get_distance(from_location, nearest.location)
for station in list_of_stations:
dis = get_distance(from_location, station.location)
if min_distance > dis:
min_distance = dis
nearest = station
return nearest
def NChgReq(self) -> int:
charging_requesters = 0
for drone in self.list_waiting_drones:
if drone.isNeedEmergencyFuel:
charging_requesters += 1
return charging_requesters
def send_drone_to_charge(self, drone: Drone): # ----->
if drone.fuel < self.min_fuel:
nearest_station = self.get_nearest_charge_station(
drone.location,
self.charging_stations_for_drones)
self.list_sent_drones.append(drone)
self.list_waiting_drones.remove(drone)
drone.isSendForEvCharge = True
# send the drone to the charging station here
print(f"Drone {drone} sent to charging station at {nearest_station}")
def check_fuel_and_send_to_charge(self): # ----->
for drone in self.list_waiting_drones:
self.send_drone_to_charge(drone)
def get_drone(self, max_try=4) -> Drone:
if max_try <= 0:
print("max try failed for get_drone")
return None
# take one time from list_waiting_drones ------->
# add to list_sent_drones
# send or return the taken item
if len(self.list_waiting_drones) == 0:
return None
else:
# lastOne = self.list_waiting_drones.pop()
self.last_one = self.list_waiting_drones.pop()
if self.last_one.fuel < self.min_fuel:
print("low fuel failed to get_drone, retry")
self.list_waiting_drones.insert(0, self.last_one)
return max_try - 1
self.list_sent_drones.append(self.last_one)
return self.last_one
def get_closest_location_from_sending_server_to_e_vehicle(self, current_vechicle_location):
min_distance = sys.maxsize
for current_server in self.charging_stations_for_drones:
distance = get_distance(current_vechicle_location, current_server.location)
if min_distance > distance:
min_distance = distance
return min_distance
def get_most_urgent_electric_vehicle(self, closest_distance_between_server_ev: int,
all_uncharged_electric_vehicles: List[Vehicle]) -> Vehicle:
final_ev = None
min_distance = sys.maxsize
for ev in all_uncharged_electric_vehicles:
g_r = self.get_closest_location_from_sending_server_to_e_vehicle(ev.location)
residual_distance = ev.residual_distance()
eq = g_r / residual_distance
if min_distance > eq:
min_distance = eq
final_ev = ev
return final_ev
def reports_termination_to_server(self, drone: Drone):
self.list_charging_drones.append(drone)
self.charge_all_waiting_drones()
def charge_all_waiting_drones(self):
# assuming the environment is not async but synchronous
for drone in self.list_charging_drones:
drone.fuel = 100
self.list_waiting_drones.insert(0, drone)
self.list_charging_drones.clear()
#staticmethod
def report_to_server(ev: Vehicle, message):
print(ev.id + " - " + message)
def get_edf(self, list_req: List[Vehicle]) -> Result:
s_uncharged_electric_vehicles = list_req
path_len = 0
n_charge_req = 0
while len(s_uncharged_electric_vehicles) > 0:
print("uncharged_ev : " + str(len(s_uncharged_electric_vehicles)))
current_uncharged_ev = s_uncharged_electric_vehicles[0]
u = self.get_drone()
if u is None:
print("no drones from any station or with min charge")
return Result(path_len, n_charge_req)
# current_uncharged_ev.location aka e----->
e = current_uncharged_ev.location
# confusion SGCS what would be the SET
g_r: int = self.get_closest_location_from_sending_server_to_e_vehicle(
e) # closest vehicle from sending location
# confusion regarding dis (loc(e), g_r) , g_r already contains the distance
e_x = self.get_most_urgent_electric_vehicle(g_r, s_uncharged_electric_vehicles)
drone_residual_distance = u.residual_distance() # Resid (U)
ev_requested_amount_of_charge = e_x.requested_amount_of_charge(u.location) # RqB2D
if drone_residual_distance < ev_requested_amount_of_charge:
self.reports_termination_to_server(u)
u = self.get_drone()
g_r_2 = self.get_closest_location_from_sending_server_to_e_vehicle(
u.location) # closest vehicle from sending location
self.reports_termination_to_server(u) # sends back the drone for charging
# ?? t is something confusing, how to include t into the equation??
rdv_loc = self.rdv_loc(e_x, u.location, 0) # t should be random
if rdv_loc is None:
self.report_to_server(e_x, "rdv location generate failed")
continue
if u.is_out_of_service_zone(rdv_loc):
self.report_to_server(e_x, "Out of Service")
continue
path_len += get_distance(u.location, rdv_loc)
u.location = rdv_loc
e_x.location = rdv_loc
# list_1 = filter(lambda x: x[3] <= 0.3 and x[2] < 5, list_1)
s_uncharged_electric_vehicles.remove(e_x)
n_charge_req = n_charge_req + 1
return Result(path_len, n_charge_req)
if __name__ == '__main__':
edf_runner = EDFGenerator()
un_charged_vehicles = [
Vehicle(1, Location(1, 1), 2),
Vehicle(2, Location(5, 10), 16),
Vehicle(3, Location(6, 10), 13),
Vehicle(4, Location(8, 11), 22),
Vehicle(5, Location(5, 6), 35),
]
edf_result = edf_runner.get_edf(un_charged_vehicles)
print("first_path_len - edf : ")
edf_result.print()
Where am I wrong? What needs fix?
I am new to python and this website
*I am having difficulties understanding why I'm getting the same value from my Car.name function
class Car:
def __init__(self, name, price):
Car.name = name
Car.price = price
I have stored some cars data in the class
Lamborghini_Centenario = Car("Lamborghini Centenario", 2500000)
Rolls_Royce_Sweptail = Car("Roll's-Royce Sweptail", 12800000)
McLarenP1LM = Car("McLaren P1 LM", 3600000)
Koenigsigg = Car("Koenigsigg CCXR Trevita", 4800000)
Lykan_Hypersport = Car("Lykan Hypersport", 3400000)
Lamborghini_Veneno_Roadster = Car("Lamborghini Veneno Roadster", 4500000)
Aston_Martin_Valkyrie = Car("Aston Martin Valkyrie", 3200000)
Bugatti_Chiron = Car("Bugatti Chiron", 2700000)
Pagani_Huayra_BC = Car("Pagani Huayra BC", 2800000)
Ferrari_Sergio = Car("Ferrari Pininfarina\n Sergio", 3000000) <--.
|
#But when I try to get the name of a random car |
#Here is the code I used vvv (Carlist is not shown here) |
#b = random.choice(carlist) | ignore arrow not part of
#b.name | code(sorry couldn't come
#it only gets the name from the bottom of the car data area. --------+ up with a name)
#I have switched it up and have only been getting the bottom value
#it might be due to the turtle module .mainloop() function
here is the whole script
Thanks in advance
import random
import turtle
import os
import time
class Car:
def __init__(self, name, price):
Car.name = name
Car.price = price
higher_true = 0
lower_true = 0
lives1 = 1
score = 0
Lamborghini_Centenario = Car("Lamborghini Centenario", 2500000)
Rolls_Royce_Sweptail = Car("Roll's-Royce Sweptail", 12800000)
McLarenP1LM = Car("McLaren P1 LM", 3600000)
Koenigsigg = Car("Koenigsigg CCXR Trevita", 4800000)
Lykan_Hypersport = Car("Lykan Hypersport", 3400000)
Lamborghini_Veneno_Roadster = Car("Lamborghini Veneno Roadster", 4500000)
Aston_Martin_Valkyrie = Car("Aston Martin Valkyrie", 3200000)
Bugatti_Chiron = Car("Bugatti Chiron", 2700000)
Pagani_Huayra_BC = Car("Pagani Huayra BC", 2800000)
Ferrari_Sergio = Car("Ferrari Pininfarina\n Sergio", 3000000)
carlist = [Lamborghini_Centenario, Rolls_Royce_Sweptail, McLarenP1LM, Koenigsigg, Lykan_Hypersport, Lamborghini_Veneno_Roadster, Aston_Martin_Valkyrie, Bugatti_Chiron, Ferrari_Sergio, Pagani_Huayra_BC]
wn = turtle.Screen()
wn.screensize(1000, 500)
wn.title("Higher or Lower: Car Price Edition")
wn.setup(1050, 650, 0, 0)
Higher = turtle.Turtle()
Higher.penup()
style = ('Courier', 55, 'italic')
Higher.color('black')
Higher.goto(130, -300)
Higher.write('Higher', font=style, align='center')
Higher.hideturtle()
Or = turtle.Turtle()
Or.penup()
Or.color('black')
Or.goto(275, -300)
Or.write('or', font=style, align='center')
Or.hideturtle()
correct = turtle.Turtle()
correct.penup()
correct.color('green')
correct.hideturtle()
Lower = turtle.Turtle()
Lower.penup()
Lower.color('black')
Lower.goto(406, -300)
Lower.write('Lower', font=style, align='center')
Lower.hideturtle()
line = turtle.Turtle()
line.goto(0, 0)
line.shape('square')
line.shapesize(500, 0.5)
line.color('black')
Lives = turtle.Turtle()
Lives.penup()
Lives.color('black')
style = ('Courier', 60, 'italic')
Lives.goto(158, 260)
Lives.write('Lives:', font=style, align='center')
Lives.hideturtle()
Scores = turtle.Turtle()
Scores.penup()
Scores.color('black')
Scores.goto(-258, 260)
Scores.write('Score:', font=style, align='center')
Scores.hideturtle()
wn.addshape(os.path.expanduser("~/Downloads/Lamborghini_Centenario.gif"))
wn.addshape(os.path.expanduser("~/Downloads/RollsRoyceSweptail.gif"))
wn.addshape(os.path.expanduser("~/Downloads/McLaren-P1-LM.gif"))
wn.addshape(os.path.expanduser("~/Downloads/Koenigsegg-CCRX-Trevita.gif"))
wn.addshape(os.path.expanduser("~/Downloads/Lamborghini-Veneno-Roadster.gif"))
wn.addshape(os.path.expanduser("~/Downloads/Aston-martin-Valkyrie.gif"))
wn.addshape(os.path.expanduser("~/Downloads/Bugatti-Chiron.gif"))
wn.addshape(os.path.expanduser("~/Downloads/Pagani-Huayra-BC.gif"))
wn.addshape(os.path.expanduser("~/Downloads/Ferrari-Pininfarina-Sergio.gif"))
wn.addshape(os.path.expanduser("~/Downloads/Lykan-Hypersport.gif"))
BugattiChiron = turtle.Turtle()
BugattiChiron.penup()
BugattiChiron.shape(os.path.expanduser("~/Downloads/Bugatti-Chiron.gif"))
BugattiChiron.goto(-300, 0)
Carname = turtle.Turtle()
Carname.penup()
Carname.color('black')
Carname.goto(-300, -200)
style1 = ('Courier', 40, 'italic')
Carname.write("Bugatti Chiron", font=style1, align='center')
Carname.hideturtle()
def screenclick(x, y):
print(x, y)
if x in range(39, 227) and y in range(-290, -250):
higher_true + 1
wn.listen()
wn.onscreenclick(screenclick)
a = Bugatti_Chiron
b = random.choice(carlist)
bname = turtle.Turtle()
bname.penup()
bname.goto(275, -200)
bname.write(b.name, font=style1, align='center')
# problem ^^^^^^ is here(it's just giving the Car.name of Ferrari_Sergio)
while higher_true == 0:
wn.update
while higher_true == 1:
if b >= a:
score = score + 1
correct.write('Correct!', font=style, align='center')
time.sleep(1.5)
correct.clear()
You need to set attributes on the object and not on the class.
class Car:
def __init__(self, name, price):
self.name = name
self.price = price
Car.name will set a name attribute on the Car class. You would be able to access the name attribute from the Car class directly without using objects like Car.name.
Setting Car.name every time updates the name attribute of the Car class and it gets updated every time you create a Car object. Hence, it retains only the last value.
I was coding for a scheduling problem where I have to schedule the trainers and the teachers at some venues. Till now I had made the code to find out the nearest three distance from the trainer. Now I want to create multiple objects of the class dynamically for all the trainers present. But I am able to create only one object. In this code I have imported the data from two csv files named mtData.csv and venueData.csvPlease help me to remove the error.
I have tried to make the objects one by one and through the loops. In both the cases I was getting the error.
import math
import csv
from math import radians, sin, cos, acos
class venue:
venueGeoLat = []
venueGeoLon = []
venueName = []
def __init__(self, name, latitude, longitude):
self.name = name
self.latitude = latitude
self.longitude = longitude
with open('venueData.csv') as csvDataFile:
csvReader = csv.reader(csvDataFile)
for row in csvReader:
venueGeoLat.append(row[1])
venueGeoLon.append(row[2])
venueName.append(row[0])
class masterTrainer:
name = None
location = None
subject = None
latitude = None
longitude = None
distanceFromVenues = []
nearestVenues = []
def __init__(self, name, location, subject, latitude, longitude):
masterTrainer.name = name
masterTrainer.location = location
masterTrainer.subject = subject
masterTrainer.latitude = latitude
masterTrainer.longitude = longitude
x = 1
y = 1
while (x < 2):
while (y < len(venue.venueGeoLat)):
masterTrainer.distanceFromVenues.append(masterTrainer.distanceCalculator(float(masterTrainer.latitude),float(masterTrainer.longitude),float(venue.venueGeoLat[y]),float(venue.venueGeoLon[y])))
y = y + 1
x = x + 1
MAX = 100000
firstmin = MAX
secmin = MAX
thirdmin = MAX
k = 0
l = 0
m = 0
for i in range(0, len(masterTrainer.distanceFromVenues)):
if masterTrainer.distanceFromVenues[i] < firstmin:
thirdmin = secmin
secmin = firstmin
firstmin = masterTrainer.distanceFromVenues[i]
k = i
elif masterTrainer.distanceFromVenues[i] < secmin:
thirdmin = secmin
secmin = masterTrainer.distanceFromVenues[i]
l = i
elif masterTrainer.distanceFromVenues[i] < thirdmin:
thirdmin = masterTrainer.distanceFromVenues[i]
m = i
masterTrainer.nearestVenues.append(venue.venueName[k+1])
masterTrainer.nearestVenues.append(venue.venueName[l+1])
masterTrainer.nearestVenues.append(venue.venueName[m+1])
def distanceCalculator(latitude1,longitude1,latitude2,longitude2):
slat = radians(latitude1)
slon = radians(longitude1)
elat = radians(latitude2)
elon = radians(longitude2)
dist = 6371.01 * acos(sin(slat)*sin(elat) + cos(slat)*cos(elat)*cos(slon - elon))
return dist
name = []
location = []
subject = []
latitude =[]
longitude = []
with open('mtData.csv') as csvDataFile:
csvReader = csv.reader(csvDataFile)
for row in csvReader:
name.append(row[0])
location.append(row[1])
subject.append(row[4])
latitude.append(row[2])
longitude.append(row[3])
mt1 = masterTrainer(name[0],location[0],subject[0],latitude[0],longitude[0])
print(mt1.nearestVenues)
mt2 = masterTrainer(name[1],location[1],subject[1],latitude[1],longitude[1])
print(mt2.nearestVenues)
print(mt1.nearestVenues)
print(mt2.distanceFromVenues)
I am getting an index error IndexError: list index out of range
Complete Traceback
<ipython-input-115-2ee0ce48bf78> in <module>
103 mt1 = masterTrainer(name[0],location[0],subject[0],latitude[0],longitude[0])
104 print(mt1.nearestVenues)
--> 105 mt2 = masterTrainer(name[1],location[1],subject[1],latitude[1],longitude[1])
106 print(mt2.nearestVenues)
107 print(mt1.nearestVenues)
<ipython-input-115-2ee0ce48bf78> in __init__(self, name, location, subject, latitude, longitude)
74 masterTrainer.nearestVenues.append(venue.venueName[k+1])
75 masterTrainer.nearestVenues.append(venue.venueName[l+1])
---> 76 masterTrainer.nearestVenues.append(venue.venueName[m+1])
77
78
IndexError: list index out of range```
I've chosen the t2.2xlarge instance with 8 CPUs and 32GiB in memory. However, I feel like the performance is the same as compared to the "free tier" version I used to run my python script on. When I look at the CPU usage on my machine, it says only 8%.
How can I utilize much more of my CPUs?
Here is the following code I'm currently running on these EC2 Instances:
def connectToDB():
databaseServerIP = "mydb.us-east-2.rds.amazonaws.com" # IP address of the MySQL database server
databaseUserName = "mydbUsername" # User name of the database server
databaseUserPassword = "mypwd" # Password for the database user
cursorType = pymysql.cursors.DictCursor
connectionInstance = pymysql.connect(host=databaseServerIP,
user=databaseUserName,
password=databaseUserPassword,
cursorclass=cursorType,
autocommit=True)
# Create a cursor object
cursorInstance = connectionInstance.cursor()
return connectionInstance, cursorInstance
def construct_each_company(tmpDF_forPeerGroup, ii):
print(tmpDF_forPeerGroup['Name'].values[ii])
finalBigDataframe = pd.DataFrame(date_generated, index = date_generated)
#symbolToCheck = tmpDF_forPeerGroup['Symbol'].values[ii]
idx = tmpDF_forPeerGroup.index[ii]
#####################
#####################
##### dataframe 1
try:
connectionInstance, cursorInstance = connectToDB()
sql = "SELECT * FROM DB1.Scores WHERE company_idx = "+str(idx)
finalBigDataframe_1 = pd.read_sql(sql, con=connectionInstance)
except:
finalBigDataframe_1 = None
#####################
#####################
##### dataframe 2
try:
connectionInstance, cursorInstance = connectToDB()
sql = "SELECT * FROM DB2.Scores WHERE company_idx = "+str(idx)
finalBigDataframe_2 = pd.read_sql(sql, con=connectionInstance)
except:
finalBigDataframe_2 = None
#####################
#####################
##### dataframe 3
try:
connectionInstance, cursorInstance = connectToDB()
sql = "SELECT * FROM DB3.Scores WHERE company_idx = "+str(idx)
finalBigDataframe_3 = pd.read_sql(sql, con=connectionInstance)
except:
finalBigDataframe_3 = None
#####################
#####################
##### dataframe 4
try:
connectionInstance, cursorInstance = connectToDB()
sql = "SELECT * FROM DB4.Scores WHERE company_idx = "+str(idx)
finalBigDataframe_4 = pd.read_sql(sql, con=connectionInstance)
except:
finalBigDataframe_4 = None
##################
##################
##################
##################
# merge for every input
# this is not right though...
tmpList_forThisCompany = [finalBigDataframe_1, finalBigDataframe_2, finalBigDataframe_3, finalBigDataframe_4]
return (ii, tmpList_forThisCompany)
def collect_result(result):
global results
results.append(result)
import multiprocessing as mp
for elem_PeerGroup in list(sorted(finalDict))[:]:
print(elem_PeerGroup)
#elem_PeerGroup = 'Africa - Banks'
########################################
### FOR ALL COMPANIES IN THIS PEER GROUP
tmpDF_forPeerGroup = finalDict[elem_PeerGroup]
if len(tmpDF_forPeerGroup)!=0:
########################
## CREATE A FINAL LIST FOR COMPANIES
#finalListForCompanies = []
########################
## CREATE DATETIME RANGE
start = datetime.strptime("01-01-2004", "%d-%m-%Y")
end = datetime.strptime("06-04-2019", "%d-%m-%Y")
date_generated = [start + timedelta(days=x) for x in range(0, (end-start).days)]
# each process will use each CPU
#pool = mp.Pool(mp.cpu_count())
pool = mp.Pool(2)
results=[]
for ii in range(0, len(tmpDF_forPeerGroup)):
pool.apply_async(construct_each_company, args=(tmpDF_forPeerGroup, ii), callback=collect_result)
pool.close()
# postpones the execution of next line of code until all processes in the queue are done.
pool.join()
# Step 5: Sort results [OPTIONAL]
results.sort(key=lambda x: x[0])
finalListForCompanies = [r for (ii, r) in results]
else:
continue
finalScores = []
# for each dataframe, NORMALIZE the companies in the PEER GROUP
for kk in range(4):
#print(kk)
tmpListForNormalisation=[]
for elem in finalListForCompanies:
tmpListForNormalisation.append(elem[kk])
dict_of_dfs = dict(enumerate(tmpListForNormalisation))
try:
dframes = pd.concat(dict_of_dfs)
except:
finalScores.append(None)
continue
dframes = dframes.iloc[:,1:]
if len(dframes)==0:
finalScores.append(None)
continue
if len(dframes)==len(dframes.groupby(level=1)):
arrayTest=[]
for k in range(len(tmpListForNormalisation)):
if (tmpListForNormalisation[k] is None) or (len(tmpListForNormalisation[k])==0):
arrayTest.append(None)
else:
arrayTest.append(tmpListForNormalisation[k])
# put the final result into a list
dict_of_dfs2 = dict(enumerate(arrayTest))
finalScores.append(dict_of_dfs2)
else:
test = dframes.groupby(level=1).pipe(lambda g: dframes.sub(g.mean(), level=1).div(g.std(), level=1))
tmpListForNormalisation2=[]
for date, new_df in test.groupby(level=0):
tmpListForNormalisation2.append(new_df)
arrayTest=[]
j=0
for k in range(len(tmpListForNormalisation)):
if (tmpListForNormalisation[k] is None) or (len(tmpListForNormalisation[k])==0):
arrayTest.append(None)
else:
arrayTest.append(tmpListForNormalisation2[j])
j+=1
test_min = test.min(level=1)
test_max = test.max(level=1)
dict_of_dfs2 = dict(enumerate(arrayTest))
def nrm(d):
_d = d
_d.index = _d.index.get_level_values(1)
NewRange = np.array([0, 100])
o = test_max - test_min
n = NewRange[1] - NewRange[0]
return (((_d - test_min) * n) / o) + NewRange[0]
for k, d in dict_of_dfs2.items():
if d is None:
continue
d.loc[:] = nrm(d).rolling(window=7).mean()
# put the final result into a list
finalScores.append(dict_of_dfs2)
# take the final MEAN for every company
for ll in range(len(tmpDF_forPeerGroup)):
namex = tmpDF_forPeerGroup['Name'].values[ll]
print("Inserting to DB...", namex)
company_idx = tmpDF_forPeerGroup['Company_idx'].values[ll]
company_symbol = tmpDF_forPeerGroup['Symbol'].values[ll]
industryName = tmpDF_forPeerGroup['GICS_Industry_Name'].values[ll]
try:
val1 = finalScores[0][ll]
except:
val1 = None
try:
val2 = finalScores[1][ll]
except:
val2 = None
try:
val3 = finalScores[2][ll]
except:
val3 = None
try:
val4 = finalScores[3][ll]
except:
val4 = None
tmpList = [val1, val2, val3, val4]
tmpDf = dict(enumerate(tmpList))
dframes = pd.concat(tmpDf)
finfin = dframes.mean(level=1)
# adjust according to its industry weights
finfin = adjustWeights(industryName, finfin)
# take data from 01.01.2007 onwards only
finfin = finfin['2007/01/01':]
#####################
# NOW PUT TO DATABASE
engine = create_engine("mysql://mydb.us-east-2.rds.amazonaws.com/"+newDatabaseName)
con = engine.connect()
finfin['timestamp'] = finfin.index
finfin['company_idx'] = [company_idx]*len(finfin)
finfin['company_symbol'] = [company_symbol]*len(finfin)
finfin.to_sql(name='Scores', con=con, if_exists='append', index=False)
I don't see why my VM is only using 8% of my CPU in this case. I don't see any error in my code as it should loop over many different companies and allocate one CPU per company.
When I run a crawler in anaconda prompt I got error:
ModuleNotFoundError: no module named 'ihs'
there is part in Pyhon file like following:
from ihs.items import IhsItem
from ihs.library import Library
which library should I install to fix this error ?
full code:
import scrapy
import re
import logging
import os
import sys
import smtplib
import urlparse
from bs4 import BeautifulSoup
from scrapy.spider import BaseSpider
from datetime import datetime
from scrapy.exceptions import CloseSpider
from ihs.items import IhsItem
from ihs.library import Library
class McewingpartnersSpider(scrapy.Spider):
name = "mcewingpartners"
lib = Library()
allowed_domains = ["mcewingpartners.com"]
start_urls = (
'http://www.mcewingpartners.com/?/rent/residential/',
)
def __init__(self):
self.log_dir = os.getcwd()
formatter = logging.Formatter('%(asctime)s %(message)s')
self.log = logging.getLogger('log-mcewingpartners')
self.log.setLevel(logging.INFO)
log_file_name = datetime.now().strftime('logs/mcewingpartners%H:%M_%d-%m-%Y.log')
ch_file = logging.FileHandler(log_file_name, 'w')
ch_file.setLevel(logging.ERROR)
ch_file.setFormatter(formatter)
self.log.addHandler(ch_file)
# add formatter to ch
ch_stream = logging.StreamHandler()
ch_stream.setFormatter(formatter)
# add ch to logger
self.log.addHandler(ch_stream)
self.log.info("mcewingpartners Ready.")
def parse(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if urlparse.urlparse(site).scheme == '':
if site.startswith('//') or site.startswith('\\'):
site = 'http:' + site
else:
site = 'http://' + site
# Property Urls
property_urls = []
_property_urls = response.xpath("//a[#class='listing-item']/#href").extract()
for _property_url in _property_urls:
token = 'self.location'
if token in _property_url[0:len(token) + 1]:
_property_url = _property_url[len(token) + 2:len(_property_url)-1]
_property_url = urlparse.urljoin(site, _property_url)
if _property_url not in property_urls:
property_urls.append(_property_url)
# check last page
if response.meta.get('prev_urls') == property_urls:
return # reached the last page
for property_url in property_urls:
yield scrapy.Request(property_url, callback = self.parse_property_urls, dont_filter = True)
#next_page_url = response.xpath("//*[#id='pageNext2']/#href").extract()
#if len(next_page_url) > 0:
# next_page_url = next_page_url[0].strip()
# if domain not in next_page_url:
# next_page_url = urlparse.urljoin(site, next_page_url)
# yield scrapy.Request(next_page_url, callback = self.parse)
except Exception as e:
err = "Error in parse: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
def parse_property_urls(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if urlparse.urlparse(site).scheme == '':
if site.startswith('//'):
site = 'http:' + site
else:
site = 'http://' + site
item = IhsItem()
item['spider_identifier'] = self.name
item['property_url'] = response.url
#Static Value
item['property_type'] = 'Other Residential'
#Static Value
item['listing_type'] = 'For Rent'
#Default value for short_description
item['short_description'] = 'Please Contact Agent'
short_description = response.xpath("//div[#class='col-md-24 property-description']//h3/text()").extract()
if len(short_description) > 0:
short_description = short_description[0].strip().encode('ascii','ignore')
item['short_description'] = short_description.strip()
#Default value for description
item['description'] = 'Please Contact Agent'
# Aggregation of multiple values from multiple xpaths
description = []
_description = response.xpath("//div[#class='col-md-24 property-description']//div[#class='inner border-bot']/p/text()").extract()
for _pd in _description:
if _pd not in description:
description.append(_pd)
if len(description) > 0:
item['description'] = ' '.join(description).encode('ascii','ignore').strip()
#Default value for price
item['price'] = 'Please Contact Agent'
price = response.xpath("//div[#class='row property-header']//div[#class='inner border-bot']//h4/text()").extract()
if len(price) > 0:
price = price[0].strip().encode('ascii','ignore')
item['price'] = price.strip()
#Default value for bedroom
item['bedroom'] = '0'
bedroom = response.xpath("(//div[#class='bbc-icon filter-bed']/text())[2]").extract()
if len(bedroom) > 0:
bedroom = bedroom[0].strip().encode('ascii','ignore')
item['bedroom'] = bedroom.strip()
#Default value for bathroom
item['bathroom'] = '0'
bathroom = response.xpath("(//div[#class='bbc-icon filter-bath']/text())[2]").extract()
if len(bathroom) > 0:
bathroom = bathroom[0].strip().encode('ascii','ignore')
item['bathroom'] = bathroom.strip()
#Default value for parking
item['parking'] = '0'
parking = response.xpath("(//div[#class='bbc-icon filter-car']/text())[2]").extract()
if len(parking) > 0:
parking = parking[0].strip().encode('ascii','ignore')
item['parking'] = parking.strip()
#Default value for photo
item['photo'] = []
# Aggregation of multiple values from multiple xpaths
photo = []
_photo = response.xpath("//div[#class='carousel-inner']//img/#src").extract()
for _pd in _photo:
if _pd not in photo:
photo.append(_pd)
if len(photo) > 0:
item['photo'] = photo
#Default value for land_area
item['land_area'] = '0'
#Default value for building_area
item['building_area'] = '0'
#Default value for inspection_date
#item['inspection_date'] = ''
#inspection_date = response.xpath("//div[#class='font13 colorthreefontcolor left valueInsp']/text()").extract()
#if len(inspection_date) > 0:
# inspection_date = inspection_date[0].strip().encode('ascii','ignore')
# item['inspection_date'] = inspection_date.strip()
#self.parse_agent_info_url(self, response)
#agent_info_url
#agent_info_url = response.xpath("//div[#id='property_staffMember']/div[#class='left staffImageContainer']/a/#href").extract()
#if len(agent_info_url) > 0:
# agent_info_url = agent_info_url[0].strip()
# agent_info_url = urlparse.urljoin(site, agent_info_url)
scrapy.Request(site, callback=self.parse_agent_info_url, dont_filter=True, meta={'item': item})
#else:
# # Default Value for items in the above branch url
# #Default value for agent_name
# item['agent_name'] = self.name.split('_')[0]
#
#Default value for agency_name
# item['agency_name'] = ''
#Default value for agent_proprietor_name
# item['agent_proprietor_name'] = self.name.split('_')[0]
#Default value for agent_licencee_number
# item['agent_licencee_number'] = self.name.split('_')[0]
#Default value for agent_licencee_name
# item['agent_licencee_name'] = self.name.split('_')[0]
#Default value for agency_logo
# item['agency_logo'] = ''
#Default value for agency_email
# item['agency_email'] = ''
#Default value for agent_email
# item['agent_email'] = 'ihomeseek#outlook.com'
#Default value for agent_mobile
# item['agent_mobile'] = ''
#Default value for agency_phone
# item['agency_phone'] = ''
#Default value for agent_fax
# item['agent_fax'] = ''
#Default value for agent_color
# item['agent_color'] = ''
yield item
except Exception as e:
err = "Error in parse_property_urls: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
def parse_agent_info_url(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if site.startswith('//') or site.startswith('\\'):
site = 'http:' + site
else:
site = 'http://' + site
item = response.meta['item']
#Default value for agent_name
item['agent_name'] = self.name.split('_')[0]
agent_name = response.xpath("//a[#class='agent-logo-inner']//img/#src").extract()
if len(agent_name) > 0:
agent_name = agent_name[0].strip().encode('ascii','ignore')
item['agent_name'] = agent_name.strip()
#Static Value
item['agency_name'] = 'McEwing Partners'
#Static Value
item['agent_proprietor_name'] = 'McEwing Partners'
#Static Value
item['agent_licencee_number'] = 'xxxxxxx'
#Static Value
item['agent_licencee_name'] = 'xxxxxxx'
#Default value for agency_logo
item['agency_logo'] = ''
agency_logo = response.xpath("//ul[#class='images']/li[#class='images']/img/#src").extract()
if len(agency_logo) > 0:
agency_logo = agency_logo[0].strip().encode('ascii','ignore')
item['agency_logo'] = agency_logo.strip()
#Default value for agency_email
item['agency_email'] = ''
agency_email = response.xpath("//*[#id='staffMember']/div[2]/span/a/text()").extract()
if len(agency_email) > 0:
agency_email = agency_email[0].strip().encode('ascii','ignore')
item['agency_email'] = agency_email.strip()
#Default value for agent_email
item['agent_email'] = '** no mail **'
agent_email = response.xpath("//*[#id='staffMember']/div[2]/span/a/text()").extract()
if len(agent_email) > 0:
agent_email = agent_email[0].strip().encode('ascii','ignore')
item['agent_email'] = agent_email.strip()
#Default value for agent_mobile
item['agent_mobile'] = ''
agent_mobile = response.xpath("//a[#class='property-staff-link']//span/text()").extract()
if len(agent_mobile) > 0:
agent_mobile = agent_mobile[0].strip().encode('ascii','ignore')
item['agent_mobile'] = agent_mobile.strip()
#Static Value
item['agency_phone'] = '03 5975 4555'
#Static Value
item['agent_fax'] = '03 5975 6444'
#Static Value
item['agent_color'] = '#3F3F3F'
yield item
except Exception as e:
err = "Error in parse_agent_info_url: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
SPIDER = McewingpartnersSpider()