When I run a crawler in anaconda prompt I got error:
ModuleNotFoundError: no module named 'ihs'
there is part in Pyhon file like following:
from ihs.items import IhsItem
from ihs.library import Library
which library should I install to fix this error ?
full code:
import scrapy
import re
import logging
import os
import sys
import smtplib
import urlparse
from bs4 import BeautifulSoup
from scrapy.spider import BaseSpider
from datetime import datetime
from scrapy.exceptions import CloseSpider
from ihs.items import IhsItem
from ihs.library import Library
class McewingpartnersSpider(scrapy.Spider):
name = "mcewingpartners"
lib = Library()
allowed_domains = ["mcewingpartners.com"]
start_urls = (
'http://www.mcewingpartners.com/?/rent/residential/',
)
def __init__(self):
self.log_dir = os.getcwd()
formatter = logging.Formatter('%(asctime)s %(message)s')
self.log = logging.getLogger('log-mcewingpartners')
self.log.setLevel(logging.INFO)
log_file_name = datetime.now().strftime('logs/mcewingpartners%H:%M_%d-%m-%Y.log')
ch_file = logging.FileHandler(log_file_name, 'w')
ch_file.setLevel(logging.ERROR)
ch_file.setFormatter(formatter)
self.log.addHandler(ch_file)
# add formatter to ch
ch_stream = logging.StreamHandler()
ch_stream.setFormatter(formatter)
# add ch to logger
self.log.addHandler(ch_stream)
self.log.info("mcewingpartners Ready.")
def parse(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if urlparse.urlparse(site).scheme == '':
if site.startswith('//') or site.startswith('\\'):
site = 'http:' + site
else:
site = 'http://' + site
# Property Urls
property_urls = []
_property_urls = response.xpath("//a[#class='listing-item']/#href").extract()
for _property_url in _property_urls:
token = 'self.location'
if token in _property_url[0:len(token) + 1]:
_property_url = _property_url[len(token) + 2:len(_property_url)-1]
_property_url = urlparse.urljoin(site, _property_url)
if _property_url not in property_urls:
property_urls.append(_property_url)
# check last page
if response.meta.get('prev_urls') == property_urls:
return # reached the last page
for property_url in property_urls:
yield scrapy.Request(property_url, callback = self.parse_property_urls, dont_filter = True)
#next_page_url = response.xpath("//*[#id='pageNext2']/#href").extract()
#if len(next_page_url) > 0:
# next_page_url = next_page_url[0].strip()
# if domain not in next_page_url:
# next_page_url = urlparse.urljoin(site, next_page_url)
# yield scrapy.Request(next_page_url, callback = self.parse)
except Exception as e:
err = "Error in parse: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
def parse_property_urls(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if urlparse.urlparse(site).scheme == '':
if site.startswith('//'):
site = 'http:' + site
else:
site = 'http://' + site
item = IhsItem()
item['spider_identifier'] = self.name
item['property_url'] = response.url
#Static Value
item['property_type'] = 'Other Residential'
#Static Value
item['listing_type'] = 'For Rent'
#Default value for short_description
item['short_description'] = 'Please Contact Agent'
short_description = response.xpath("//div[#class='col-md-24 property-description']//h3/text()").extract()
if len(short_description) > 0:
short_description = short_description[0].strip().encode('ascii','ignore')
item['short_description'] = short_description.strip()
#Default value for description
item['description'] = 'Please Contact Agent'
# Aggregation of multiple values from multiple xpaths
description = []
_description = response.xpath("//div[#class='col-md-24 property-description']//div[#class='inner border-bot']/p/text()").extract()
for _pd in _description:
if _pd not in description:
description.append(_pd)
if len(description) > 0:
item['description'] = ' '.join(description).encode('ascii','ignore').strip()
#Default value for price
item['price'] = 'Please Contact Agent'
price = response.xpath("//div[#class='row property-header']//div[#class='inner border-bot']//h4/text()").extract()
if len(price) > 0:
price = price[0].strip().encode('ascii','ignore')
item['price'] = price.strip()
#Default value for bedroom
item['bedroom'] = '0'
bedroom = response.xpath("(//div[#class='bbc-icon filter-bed']/text())[2]").extract()
if len(bedroom) > 0:
bedroom = bedroom[0].strip().encode('ascii','ignore')
item['bedroom'] = bedroom.strip()
#Default value for bathroom
item['bathroom'] = '0'
bathroom = response.xpath("(//div[#class='bbc-icon filter-bath']/text())[2]").extract()
if len(bathroom) > 0:
bathroom = bathroom[0].strip().encode('ascii','ignore')
item['bathroom'] = bathroom.strip()
#Default value for parking
item['parking'] = '0'
parking = response.xpath("(//div[#class='bbc-icon filter-car']/text())[2]").extract()
if len(parking) > 0:
parking = parking[0].strip().encode('ascii','ignore')
item['parking'] = parking.strip()
#Default value for photo
item['photo'] = []
# Aggregation of multiple values from multiple xpaths
photo = []
_photo = response.xpath("//div[#class='carousel-inner']//img/#src").extract()
for _pd in _photo:
if _pd not in photo:
photo.append(_pd)
if len(photo) > 0:
item['photo'] = photo
#Default value for land_area
item['land_area'] = '0'
#Default value for building_area
item['building_area'] = '0'
#Default value for inspection_date
#item['inspection_date'] = ''
#inspection_date = response.xpath("//div[#class='font13 colorthreefontcolor left valueInsp']/text()").extract()
#if len(inspection_date) > 0:
# inspection_date = inspection_date[0].strip().encode('ascii','ignore')
# item['inspection_date'] = inspection_date.strip()
#self.parse_agent_info_url(self, response)
#agent_info_url
#agent_info_url = response.xpath("//div[#id='property_staffMember']/div[#class='left staffImageContainer']/a/#href").extract()
#if len(agent_info_url) > 0:
# agent_info_url = agent_info_url[0].strip()
# agent_info_url = urlparse.urljoin(site, agent_info_url)
scrapy.Request(site, callback=self.parse_agent_info_url, dont_filter=True, meta={'item': item})
#else:
# # Default Value for items in the above branch url
# #Default value for agent_name
# item['agent_name'] = self.name.split('_')[0]
#
#Default value for agency_name
# item['agency_name'] = ''
#Default value for agent_proprietor_name
# item['agent_proprietor_name'] = self.name.split('_')[0]
#Default value for agent_licencee_number
# item['agent_licencee_number'] = self.name.split('_')[0]
#Default value for agent_licencee_name
# item['agent_licencee_name'] = self.name.split('_')[0]
#Default value for agency_logo
# item['agency_logo'] = ''
#Default value for agency_email
# item['agency_email'] = ''
#Default value for agent_email
# item['agent_email'] = 'ihomeseek#outlook.com'
#Default value for agent_mobile
# item['agent_mobile'] = ''
#Default value for agency_phone
# item['agency_phone'] = ''
#Default value for agent_fax
# item['agent_fax'] = ''
#Default value for agent_color
# item['agent_color'] = ''
yield item
except Exception as e:
err = "Error in parse_property_urls: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
def parse_agent_info_url(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if site.startswith('//') or site.startswith('\\'):
site = 'http:' + site
else:
site = 'http://' + site
item = response.meta['item']
#Default value for agent_name
item['agent_name'] = self.name.split('_')[0]
agent_name = response.xpath("//a[#class='agent-logo-inner']//img/#src").extract()
if len(agent_name) > 0:
agent_name = agent_name[0].strip().encode('ascii','ignore')
item['agent_name'] = agent_name.strip()
#Static Value
item['agency_name'] = 'McEwing Partners'
#Static Value
item['agent_proprietor_name'] = 'McEwing Partners'
#Static Value
item['agent_licencee_number'] = 'xxxxxxx'
#Static Value
item['agent_licencee_name'] = 'xxxxxxx'
#Default value for agency_logo
item['agency_logo'] = ''
agency_logo = response.xpath("//ul[#class='images']/li[#class='images']/img/#src").extract()
if len(agency_logo) > 0:
agency_logo = agency_logo[0].strip().encode('ascii','ignore')
item['agency_logo'] = agency_logo.strip()
#Default value for agency_email
item['agency_email'] = ''
agency_email = response.xpath("//*[#id='staffMember']/div[2]/span/a/text()").extract()
if len(agency_email) > 0:
agency_email = agency_email[0].strip().encode('ascii','ignore')
item['agency_email'] = agency_email.strip()
#Default value for agent_email
item['agent_email'] = '** no mail **'
agent_email = response.xpath("//*[#id='staffMember']/div[2]/span/a/text()").extract()
if len(agent_email) > 0:
agent_email = agent_email[0].strip().encode('ascii','ignore')
item['agent_email'] = agent_email.strip()
#Default value for agent_mobile
item['agent_mobile'] = ''
agent_mobile = response.xpath("//a[#class='property-staff-link']//span/text()").extract()
if len(agent_mobile) > 0:
agent_mobile = agent_mobile[0].strip().encode('ascii','ignore')
item['agent_mobile'] = agent_mobile.strip()
#Static Value
item['agency_phone'] = '03 5975 4555'
#Static Value
item['agent_fax'] = '03 5975 6444'
#Static Value
item['agent_color'] = '#3F3F3F'
yield item
except Exception as e:
err = "Error in parse_agent_info_url: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
SPIDER = McewingpartnersSpider()
Related
My electric vehicle charging algorithm aims at returning the shortest path length by a charging drone. The drone will meet the electric vehicle at a rendezvous point and charge the vehicle. The vehicle selection is done based on the charging urgency of the vehicles. The total path length is calculated when all vehicles are charged based on their urgency. The pseudocode is:
EDF(List_Req)
SunchrgEV = List_Req
Pathlen = 0
Nchgreq = 0
while(SunchrgEV = 0):
U obtains the updated location of all ei element of SunchrgEV via message exchange
S'gcs = set of GCS element of Sgcs
Gr = min {Eucleadian distance between Loc(ei) and Loc(Gj)} //The closest GCS
ex = min [{Eucleadian distance between Loc(ei) and Loc(Gr)}/ResidDist(ei)] // most urgent EV
if SoC(U)<RqB2D(ex) // then U(the drone) itself needs charge and it reports its termination to the server
// the server dispatches another U(drone) to charge the remaining EVs (ei)
Gr = min {Eucleadian distance between Loc(U) and Loc(Gj)} //The closest GCS to U where U goes to full charge
end if
t = tcurrent
// Finding rendezvous point where ex and U meets
RdvLoc = FindRdvLoc(ex, Loc(ex), Loc(U),t)
if RdvLoc is out of service area of U then
Report2server(ex,'Outofservicearea')
continue
end if
Pathlen += Dist2(Loc(U),RdvLoc)
U sends RdvLoc to ex and flies to RdvLoc
if U reaches ex in ChgRng(U) then
{Pathlen += Charge(ex, RdvLoc)
Nchgreq ++
}
else
Report2server(ex, 'Outofservicearea')
endif
SunchrgEV -= ex
do
Update (Loc(U))
Gr = min {Eucleadian distance between Loc(U) and Loc(Gj)} //U returns to Gr
Pathlen += Dist2(Loc(U), Loc(Gr))
return Pathlen and Nchgreq
//Compute the expected location where U meets e on Map(e)
def FindRdvLoc(e,Le,Lu,t):
Compute X = (x,y) on Rte(e,t) so that {Dist(e,X)/Speed(e,t)} = {Dist(U, X-ChgRng(U))/Speed(U)}
return X
def Charge(ei, RdvLoc):
U starts to charge ei and follows Rte(ei,t)
eLoc = FindLoc(Map(ei), Loc(ei), ChgTime(ei), Speed(ei,t))
return DistM(Map(ei), RdvLoc, eLoc)
The code that I have written so far gives the same output regardless of the input. The code is:
import math
import sys
from typing import List, Any
class Location:
x = 0
y = 0
def __init__(self, x, y):
self.x = x
self.y = y
class Result:
path_len: int = 0
n_charge_request: int = 0
def __init__(self, path_len, n_charge_request):
self.path_len = path_len
self.n_charge_request = n_charge_request
def to_string(self):
return "path len : " + str(self.path_len) + ", n_charge_request : " + str(self.n_charge_request)
def print(self):
print(self.to_string())
#SGCs
class ChargingStation:
def __init__(self, location: Location, fuel):
self.location = location
self.fuel = fuel
#EVs(ex)
class Vehicle:
location = Location(0, 0)
fuel = 1
isNeedEmergencyFuel = False
per_fuel_distance_travel = 2
def __init__(self, id, location, fuel):
self.id = id
self.location = location
self.fuel = fuel
# Resid (U)
def residual_distance(self):
return self.fuel * self.per_fuel_distance_travel # assuming each watt or kw fuel will yield to 2 killos or milies
# RqB2D
def requested_amount_of_charge(self, nearest_charge_station_location: Location) -> int:
distance = get_distance(self.location, nearest_charge_station_location)
cover = self.fuel * self.per_fuel_distance_travel
diff = math.fabs(distance - cover)
if diff > 0:
needed_fuel = diff / self.per_fuel_distance_travel + 2
return needed_fuel
return 0
# U(i)
class Drone:
location = Location(0, 0)
fuel = 0
isFlying = False
isSendForEvCharge = False
per_fuel_distance_travel = 20
serving_radius = 15
G = [
ChargingStation(Location(20, 10), 50),
ChargingStation(Location(50, 80), 40),
ChargingStation(Location(30, 30), 60)
]
def __init__(self, location, fuel):
self.location = location
self.fuel = fuel
# Resid (U)
def residual_distance(self):
return self.fuel * self.per_fuel_distance_travel # assuming each unit of fuel will yield to 2 kilos or miles
def is_out_of_service_zone(self, vehicle_location: Location): # ----->
distance = get_distance(self.location, vehicle_location)
return distance > self.serving_radius
#staticmethod
def get_distance(from_location, to_location):
x_dis = to_location.x - from_location.x
y_dis = to_location.y - from_location.y
x_dis_sqr = math.pow(x_dis, 2)
y_dis_sqr = math.pow(y_dis, 2)
final_dis_sum = x_dis_sqr + y_dis_sqr
final = math.sqrt(final_dis_sum)
return final
class EDFGenerator:
min_fuel = 50
charging_stations_for_drones = [ChargingStation(Location(2, 10), 80), ChargingStation(Location(2, 10), 50),
ChargingStation(Location(2, 10), 100)]
list_waiting_drones = [Drone(Location(5, 10), 50), Drone(Location(2, 10), 50), Drone(Location(2, 10), 50)]
list_sent_drones = []
list_charging_drones = []
def __init__(self):
pass
def rdv_loc(self, ei: Vehicle, drone_location: Location, t: int) -> Location | None:
needed_fuel = ei.requested_amount_of_charge(drone_location)
nearest_charge_station = self.get_nearest_charge_station(ei.location, self.charging_stations_for_drones)
needed_fuel_at_nearest_station = needed_fuel / ei.per_fuel_distance_travel
if nearest_charge_station.fuel < needed_fuel_at_nearest_station:
return None
else:
return nearest_charge_station.location
#staticmethod
def get_nearest_charge_station(from_location: Location, list_of_stations: List[ChargingStation]) -> ChargingStation:
nearest = list_of_stations[0]
min_distance = get_distance(from_location, nearest.location)
for station in list_of_stations:
dis = get_distance(from_location, station.location)
if min_distance > dis:
min_distance = dis
nearest = station
return nearest
def NChgReq(self) -> int:
charging_requesters = 0
for drone in self.list_waiting_drones:
if drone.isNeedEmergencyFuel:
charging_requesters += 1
return charging_requesters
def send_drone_to_charge(self, drone: Drone): # ----->
if drone.fuel < self.min_fuel:
nearest_station = self.get_nearest_charge_station(
drone.location,
self.charging_stations_for_drones)
self.list_sent_drones.append(drone)
self.list_waiting_drones.remove(drone)
drone.isSendForEvCharge = True
# send the drone to the charging station here
print(f"Drone {drone} sent to charging station at {nearest_station}")
def check_fuel_and_send_to_charge(self): # ----->
for drone in self.list_waiting_drones:
self.send_drone_to_charge(drone)
def get_drone(self, max_try=4) -> Drone:
if max_try <= 0:
print("max try failed for get_drone")
return None
# take one time from list_waiting_drones ------->
# add to list_sent_drones
# send or return the taken item
if len(self.list_waiting_drones) == 0:
return None
else:
# lastOne = self.list_waiting_drones.pop()
self.last_one = self.list_waiting_drones.pop()
if self.last_one.fuel < self.min_fuel:
print("low fuel failed to get_drone, retry")
self.list_waiting_drones.insert(0, self.last_one)
return max_try - 1
self.list_sent_drones.append(self.last_one)
return self.last_one
def get_closest_location_from_sending_server_to_e_vehicle(self, current_vechicle_location):
min_distance = sys.maxsize
for current_server in self.charging_stations_for_drones:
distance = get_distance(current_vechicle_location, current_server.location)
if min_distance > distance:
min_distance = distance
return min_distance
def get_most_urgent_electric_vehicle(self, closest_distance_between_server_ev: int,
all_uncharged_electric_vehicles: List[Vehicle]) -> Vehicle:
final_ev = None
min_distance = sys.maxsize
for ev in all_uncharged_electric_vehicles:
g_r = self.get_closest_location_from_sending_server_to_e_vehicle(ev.location)
residual_distance = ev.residual_distance()
eq = g_r / residual_distance
if min_distance > eq:
min_distance = eq
final_ev = ev
return final_ev
def reports_termination_to_server(self, drone: Drone):
self.list_charging_drones.append(drone)
self.charge_all_waiting_drones()
def charge_all_waiting_drones(self):
# assuming the environment is not async but synchronous
for drone in self.list_charging_drones:
drone.fuel = 100
self.list_waiting_drones.insert(0, drone)
self.list_charging_drones.clear()
#staticmethod
def report_to_server(ev: Vehicle, message):
print(ev.id + " - " + message)
def get_edf(self, list_req: List[Vehicle]) -> Result:
s_uncharged_electric_vehicles = list_req
path_len = 0
n_charge_req = 0
while len(s_uncharged_electric_vehicles) > 0:
print("uncharged_ev : " + str(len(s_uncharged_electric_vehicles)))
current_uncharged_ev = s_uncharged_electric_vehicles[0]
u = self.get_drone()
if u is None:
print("no drones from any station or with min charge")
return Result(path_len, n_charge_req)
# current_uncharged_ev.location aka e----->
e = current_uncharged_ev.location
# confusion SGCS what would be the SET
g_r: int = self.get_closest_location_from_sending_server_to_e_vehicle(
e) # closest vehicle from sending location
# confusion regarding dis (loc(e), g_r) , g_r already contains the distance
e_x = self.get_most_urgent_electric_vehicle(g_r, s_uncharged_electric_vehicles)
drone_residual_distance = u.residual_distance() # Resid (U)
ev_requested_amount_of_charge = e_x.requested_amount_of_charge(u.location) # RqB2D
if drone_residual_distance < ev_requested_amount_of_charge:
self.reports_termination_to_server(u)
u = self.get_drone()
g_r_2 = self.get_closest_location_from_sending_server_to_e_vehicle(
u.location) # closest vehicle from sending location
self.reports_termination_to_server(u) # sends back the drone for charging
# ?? t is something confusing, how to include t into the equation??
rdv_loc = self.rdv_loc(e_x, u.location, 0) # t should be random
if rdv_loc is None:
self.report_to_server(e_x, "rdv location generate failed")
continue
if u.is_out_of_service_zone(rdv_loc):
self.report_to_server(e_x, "Out of Service")
continue
path_len += get_distance(u.location, rdv_loc)
u.location = rdv_loc
e_x.location = rdv_loc
# list_1 = filter(lambda x: x[3] <= 0.3 and x[2] < 5, list_1)
s_uncharged_electric_vehicles.remove(e_x)
n_charge_req = n_charge_req + 1
return Result(path_len, n_charge_req)
if __name__ == '__main__':
edf_runner = EDFGenerator()
un_charged_vehicles = [
Vehicle(1, Location(1, 1), 2),
Vehicle(2, Location(5, 10), 16),
Vehicle(3, Location(6, 10), 13),
Vehicle(4, Location(8, 11), 22),
Vehicle(5, Location(5, 6), 35),
]
edf_result = edf_runner.get_edf(un_charged_vehicles)
print("first_path_len - edf : ")
edf_result.print()
Where am I wrong? What needs fix?
I've got a weather display for a Pi which uses pygame to display the data
The issue I have is when the internet dies for any reason there is no update and the display shows blank data
What I'd like to do is if there is no update then it keeps the previous data on the screen
is this possible?
This is an example of the code that displays the data
if forecastData.status == forecast.STATUS_OK:
ren = font.render("Solar Radiation: {} W/m2".format(forecastData.solar_radiation), 1, pg.Color('black'), pg.Color(185,208,240))
else:
ren = font.render("Solar Radiation: ", 1, pg.Color('black'), pg.Color(185,208,240))
screen.blit(ren, (5*HRES//1600, 430*VRES//900-ren.get_height()//2))
When there is no update this displays Solar Radiation: only - I'd like this to use the previous data - ie DONT update this section of the display
It seems like what you should do is check the status before updating self.data in your Forecast class:
#Now make it available to outside world.
if data.status == STATUS_OK:
self.lock.acquire()
self.data = data
self.lock.release()
Can't be sure as it's not possible to run any of your code samples.
Here is a minimal example that uses a thread to simulate data retrieval with the occasional error.
import random
import threading
import time
import pygame
class ForecastThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self.start() # starts on instantiation!
def retrieve_data(self):
global data
# make request to forecast service and parse response
response = random.randint(1, 5)
if response == 1:
# Request failed
data = """(○o◌!*^##!#"""
else: # success
data = random.choice(("Cloudy", "Rainy", "Stormy", "Sunny", "Windy"))
def run(self):
while True:
if not running: # exit thread if main loop exits
return
else:
self.retrieve_data()
time.sleep(1) # sleep for a second
WIDTH = 480
HEIGHT = 240
FPS = 30
random.seed(98765432)
pygame.init()
window = pygame.display.set_mode((WIDTH, HEIGHT))
clock = pygame.time.Clock()
# grab the first installed font
sys_font = pygame.font.SysFont(pygame.font.get_fonts()[0], 100)
data = "" # create globals before thread
running = True
forecaster_thread = ForecastThread() # note: thread is auto-starting
while running:
for event in pygame.event.get():
if event.type == pygame.QUIT:
running = False
elif event.type == pygame.KEYUP:
if event.key == pygame.K_ESCAPE:
running = False
# update title bar
pygame.display.set_caption(f"Forecaster FPS: {clock.get_fps():.1f}")
# fill background to clear
window.fill(pygame.Color("lightsteelblue2"))
# create image
image = sys_font.render(data, True, pygame.Color("black"))
window.blit(image, (50, 50)) # blit close to centre
# show surface
pygame.display.update()
# limit frames
clock.tick(FPS)
pygame.quit()
Running this code will initially display Cloudy then change to the gibberish data indicating a failed request.
If you change the retrieve_data() function to not update data on failure, then no gibberish will be displayed.
def retrieve_data(self):
global data
# make request to forecast service and parse response
response = random.randint(1, 5)
if response == 1:
# Request failed
print("Failed Request") # log the error
else: # success
data = random.choice(("Cloudy", "Rainy", "Stormy", "Sunny", "Windy"))
"import random" I have tried your solution so many ways but it always fails Does this help?... this is the full code to collect the forecast
import json
import urllib.request
import urllib.error
import time
from datetime import datetime
from datetime import timedelta
import pdb
import threading
import pygame as pg
import logging
import os.path
import socket
FORECAST_URL = "https://swd.weatherflow.com/swd/rest/better_forecast?api_key=20c70eae-e62f-4d3b-b3a4-8586e90f3ac8&station_id=44303&lat=53.440&lon=-2.105"
TIMEOUT = 15
STATUS_TIMEOUT = "Timeout"
STATUS_OK = "OK"
def nullis0hook(d):
"""This is a hook for the JSON decoder, replacing null with 0."""
for k in d.keys():
if d[k]==None:
d[k]=0
return d
class ForecastData:
def __init__(self):
self.status = STATUS_OK
self.conditions=""
self.iconnow = ""
self.updateTime="946684800"
self.tempnow=0
self.templow=30
self.temphigh=-10
self.sea_level_pressure=0
self.station_pressure=0
self.pressure_trend=""
self.relative_humidity=0
self.wind_avg=0
self.wind_direction_cardinal=""
self.angle=0
self.wind_gust=0
self.solar_radiation=0
self.uv=0
self.feels_like=0
self.dew_point=0
self.wet_bulb_temperature=""
self.delta_t=0
self.air_density=0
self.lightning_strike_last_distance="0"
self.lightning1=""
self.lightning_strike_last_epoch="946684800"
self.precip_accum_local_yesterday="0"
self.precip_accum_local_day="0"
self.condday=[""]*6
self.icon=[""]*6
self.iconday_filename = [os.path.join("images",os.path.join("forecast_icons","--_"))]*6
self.iconday = [pg.image.load(os.path.join("images",os.path.join("forecast_icons","--_1.bmp"))).convert()]*6
self.thighday=[0]*6
self.tlowday=[0]*6
self.sunriseday=[""]*6
self.sunsetday=[""]*6
self.precprday=[0]*6
self.precpiconday=[""]*6
self.preciptypeday=[""]*6
self.conditionshour=[""]*9
self.iconhour=[""]*9
self.precipprhour=[""]*9
self.preciptypehour=[""]*9
self.feelslikehour=[0]*9
self.conditionshour=[""]*9
self.iconhour=[""]*9
self.precipprhour=[""]*9
self.preciptypehour=[""]*9
self.feelslikehour=[0]*9
self.airtemphour=[0]*9
self.windavghour=[0]*9
self.forecast_icon_filename = os.path.join("images",os.path.join("weather_icons","--_.bmp"))
self.forecast_icon = pg.image.load(self.forecast_icon_filename).convert()
self.kia = pg.image.load(os.path.join("images",os.path.join("weather_icons","kia.png"))).convert()
#The Forecast class retrieves the weatherflow.com forecast and extracts relevant forecast data.
#Note: pygame display must be initialized before constructing a Forecast object
class ForeCastHour:
def __init__(self):
self.conditions = [""]*6
self.icon = ["--_"]*6
self.precipr = [""]*6
self.precip_type = [""]*6
self.feels_like = [""]*6
class Forecast:
def __init__(self):
self.data = ForecastData()
self.lock = threading.Lock()
self.max_wind_gust=0
self.templow=30
self.temphigh=-10
self.updateTime = ""
def getData(self):
"""Get most recently retrieved consistent set of data."""
self.lock.acquire()
data = self.data
self.lock.release()
return data
def midnightReset(self):
"""Reset any values that require resetting at midnight"""
self.lock.acquire()
self.data.wind_gust=0
self.lock.release()
def update(self):
"""Update the forecast data"""
data = ForecastData()
try:
req = urllib.request.Request(FORECAST_URL)
with urllib.request.urlopen(req, timeout=TIMEOUT) as response:
raw = response.read()
forecast_json = json.loads(raw, object_hook=nullis0hook)
data.status = STATUS_OK
self.updateTime = forecast_json["current_conditions"]["time"]
data.updateTime = time.strftime("%H:%M:%S", time.localtime(self.updateTime))
data.conditions = forecast_json["current_conditions"]["conditions"]
iconnow = forecast_json["current_conditions"]["icon"]
if iconnow == "null":
iconnow = "--_" #icon replacement
if iconnow == "":
iconnow = "--_" #icon replacement
data.iconnow_filename = os.path.join("images",os.path.join("weather_icons",iconnow+".bmp"))
if os.path.exists(data.iconnow_filename):
data.iconnow = pg.image.load(data.iconnow_filename).convert()
else:
logging.warning("Weather icon file {} not found.".format(data.iconnow_filename))
data.tempnow = forecast_json["current_conditions"]["air_temperature"]
if data.tempnow < self.templow:
self.templow = data.tempnow
if data.tempnow > self.temphigh:
self.temphigh = data.tempnow
data.templow = self.templow
data.temphigh = self.temphigh
data.sea_level_pressure = forecast_json["current_conditions"]["sea_level_pressure"]
data.station_pressure = forecast_json["current_conditions"]["station_pressure"]
data.pressure_trend = forecast_json["current_conditions"]["pressure_trend"]
data.relative_humidity = forecast_json["current_conditions"]["relative_humidity"]
data.wind_avg = forecast_json["current_conditions"]["wind_avg"]* 2.23694 #Convert mps to mph
data.wind_gust = forecast_json["current_conditions"]["wind_gust"] * 2.23694 #Convert mps to mph
data.angle = forecast_json["current_conditions"]["wind_direction"]
if data.angle <= 180:
data.angle = data.angle + 180
else:
data.angle = data.angle - 180
data.wind_direction_cardinal = forecast_json["current_conditions"]["wind_direction_cardinal"]
data.solar_radiation = forecast_json["current_conditions"]["solar_radiation"]
data.uv = forecast_json["current_conditions"]["uv"]
data.feels_like = forecast_json["current_conditions"]["feels_like"]
lightning_strike_last_distance = forecast_json["current_conditions"].get("lightning_strike_last_distance", 0)
lightning1 = lightning_strike_last_distance*0.621371 #Convert kph to mph
data.lightning_strike_last_distance = "{0:.1f} miles away ".format(lightning1)
lightning_strike_last_epoch = forecast_json["current_conditions"].get("lightning_strike_last_epoch")
data.lightning_strike_last_epoch = time.strftime("%d %b", time.localtime(lightning_strike_last_epoch))
data.precip_accum_local_yesterday = forecast_json["current_conditions"]["precip_accum_local_yesterday"]
data.precip_accum_local_day = forecast_json["current_conditions"]["precip_accum_local_day"]
for day in range(6):
data.sunriseday[day] = forecast_json["forecast"]["daily"][day]["sunrise"]
data.sunriseday[day] = time.strftime("%H:%M:%S", time.localtime(data.sunriseday[day]))
data.sunsetday[day] = forecast_json["forecast"]["daily"][day]["sunset"]
data.sunsetday[day] = time.strftime("%H:%M:%S", time.localtime(data.sunsetday[day]))
data.condday[day] = forecast_json["forecast"]["daily"][day]["conditions"]
icon = forecast_json["forecast"]["daily"][day]["icon"]
data.iconday_filename[day] = os.path.join("images",os.path.join("forecast_icons",icon+"1.bmp"))
if os.path.exists(data.iconday_filename[day]):
iconimage = pg.image.load(data.iconday_filename[day]).convert()
data.iconday[day] = iconimage
else:
logging.warning("Forecast icon file {} not found.".format(data.iconday_filename[day]))
data.thighday[day] = forecast_json["forecast"]["daily"][day]["air_temp_high"]
data.tlowday[day] = forecast_json["forecast"]["daily"][day]["air_temp_low"]
data.precprday[day] = forecast_json["forecast"]["daily"][day]["precip_probability"]
if data.precprday[day] != 0:
data.precpiconday[day] = forecast_json["forecast"]["daily"][day]["precip_icon"]
data.preciptypeday[day] = forecast_json["forecast"]["daily"][day]["precip_type"]
data.forecast_icon_filename = os.path.join("images",os.path.join("weather_icons",iconnow+".bmp"))
if os.path.exists(data.forecast_icon_filename):
data.forecast_icon = pg.image.load(data.forecast_icon_filename).convert()
else:
logging.warning("Forecast icon file {} not found.".format(data.forecast_icon_filename))
for hours in range(9):
ps = forecast_json["forecast"]["hourly"][hours]["conditions"]
if ps == "Wintry Mix Possible":
ps = "Winty-P"
if ps == "Wintry Mix Likely":
ps = "Winty-L"
if ps == "Rain Likely":
ps = "Rain-L"
if ps == "Rain Possible":
ps ="Rain-P"
if ps == "Snow Possible":
ps = "Snow-P"
if ps == "Thunderstorms Likely":
ps = "ThundrL"
if ps == "Thunderstorms Possible":
ps = "ThundrP"
if ps == "Partly Cloudy":
ps = "Clouds"
if ps == "Very Light Rain":
ps = "drizzle"
data.conditionshour[hours] = "{}".format(ps)
data.iconhour[hours] = forecast_json["forecast"]["hourly"][hours]["icon"]
pp = forecast_json["forecast"]["hourly"][hours]["precip_probability"]
data.precipprhour[hours] = "{}%".format(pp)
if pp == 0:
data.preciptypehour[hours] = "0"
else:
data.preciptypehour[hours] = forecast_json["forecast"]["hourly"][hours]["precip_type"]
data.feelslikehour[hours] = "{} C".format(forecast_json["forecast"]["hourly"][hours]["feels_like"])
data.airtemphour[hours] = forecast_json["forecast"]["hourly"][hours]["air_temperature"]
data.windavghour[hours] = forecast_json["forecast"]["hourly"][hours]["wind_avg"]*0.621371 #Convert kph to mph
#datetime object containing current date and time
now = datetime.now()
data.updateTime = now.strftime("%H:%M:%S")
self.updateTime = now
except (socket.timeout, socket.gaierror, urllib.error.URLError, json.decoder.JSONDecodeError, KeyError):
logging.warning("Error retrieving forecast data")
#declare timeout only after timeout period
if datetime.now() - self.updateTime > timedelta(seconds=TIMEOUT):
data.status = STATUS_TIMEOUT
data.updateTime = self.data.updateTime #Use old update time value
else: #If timeout period has not elapsed yet, use previous data
logging.info("Not timing out yet")
self.data = self.data #Use old value
if datetime.now() - self.updateTime > timedelta(seconds=TIMEOUT):
data.status = STATUS_TIMEOUT
else: #If timeout period has not elapsed yet, use previous data
logging.info("Not timing out yet")
data = self.data
#Now make it available to outside world.
self.lock.acquire()
self.data = data
self.lock.release()
And this is what works when there is internet, but when the internet goes off it doesn't use the old data
'''THIS IS THE FORECAST SECTION THAT WORKS WHEN THERE IS INTENET AND FORECAST UPDATES
BUT WHEN THERE IS NO UPDATE IT JUST DISPLAYS THE TEXT AND DOES NOT USE THE OLD DATA VALUE'''
if forecastData.status == forecast.STATUS_OK:
ren = font.render("battery voltage : " + "{} V".format(forecastData.battery), 1, pg.Color('white'), pg.Color(162, 160, 160))
else:
ren = font.render("", 1, pg.Color('white'), pg.Color(162, 160, 160))
screen.blit(ren, (700*HRES//1600, VRES//60))
if forecastData.status == forecast.STATUS_OK:
ren = font.render("Conditions: {} ".format(forecastData.conditions), 1, pg.Color('black'), pg.Color(185,208,240))
else:
ren = font.render("Conditions: ", 1, pg.Color('black'), pg.Color(185,208,240))
screen.blit(ren, (5*HRES//1600, 70*VRES//900-ren.get_height()//2))
'''THIS IS THE ERROR MESSAGE I GET LOGGED'''
2021-04-15 13:28:41,057 - root - INFO - Updating forecast.
2021-04-15 13:28:41,087 - root - WARNING - Error retrieving forecast data
2021-04-15 13:29:11,049 - root - WARNING - Previous every-minute thread still running. Not relaunching.
2021-04-15 13:29:26,602 - root - INFO - t key pressed. Toggle fullscreen.
2021-04-15 13:29:41,085 - root - WARNING - Previous every-minute thread still running. Not relaunching.
2021-04-15 13:30:11,089 - root - WARNING - Previous every-minute thread still running. Not relaunching.
'''THIS IS THE FUNCTION THAT THE ERROR REFERS TO'''
def everyMinuteThreadFunction():
"""This thread function executes once every minute."""
global initialWeatherUpdateReceived, everyMinuteThreadRunning
everyMinuteThreadRunning = True
assert(forecastObj)
checkMidnightRollOver()
logging.info("Updating forecast.")
forecastObj.update()
forecastData = forecastObj.getData()
if forecastData.status == forecast.STATUS_OK:
logging.info("Forecast data: {}".format(vars(forecastData)))
#The first time we pass here is a good time to kick off the five minute task. We now have our first
#forecast evice data available
if not initialWeatherUpdateReceived:
#program a periodic timer used to kick off the everyFiveMinutesThreadFunction.
pg.time.set_timer(EVERY_FIVE_MINUTES_THREAD_FUNCTION_EVENT, 5*60000)
#Kick off the task now, for the initial interval.
t = threading.Thread(target=everyFiveMinutesThreadFunction, args=())
t.daemon = True
t.start()
updateGauge()
initialWeatherUpdateReceived = True
everyMinuteThreadRunning = False
I want to delete seed and insert specific hex key with input
import hashlib
import random
import time
import base58
from .util import doublehash256
class Key:
def __init__(self, seed=None):
self.hash = ''
self.hashdigest = ''
self.hex = ''
self.wif = ''
self.wif_c = ''
def _seed(self):
t = int(time.time())
return str(random.getrandbits(3000) - t)
def generate(self, seed=None):
self._generate_raw(seed)
self._generate_hex()
self._generate_wif()
return {'hex': self.hex, 'wif': self.wif, 'wifc': self.wif_c}
def _generate_raw(self, seed=None):
if (seed == None): seed = self._seed()
self.hash = hashlib.sha256(seed.encode())
self.hashdigest = self.hash.digest()
def _generate_hex(self):
self.hex = self.hash.hexdigest()
def _generate_wif(self):
prefix = b'\x80'
suffix = b'\x01'
d = prefix + self.hashdigest
checksum = doublehash256(d).digest()[:4]
checksum_c = doublehash256(d + suffix).digest()[:4]
self.wif = base58.b58encode(d + checksum).decode('utf-8')
self.wif_c = base58.b58encode(d + suffix + checksum_c).decode('utf-8')
def __str__(self):
return """Private Key HEX: %s
\rPrivate Key WIF: %s\r
\rPrivate Key WIF compressed: %s
""" % (self.hex, self.wif, self.wif_c)
like this one
An instance of the Rabbit class is created in a for loop and given a datetime object as init variable
but all instances get the same datetime value.
I expected a time difference for each instance; what am I missing in understanding here?
from datetime import datetime
class Rabbit_birth:
num_eyes = 2 # <- Class variable
num_legs = 4
num_ears = 2
num_tails = 1
num_of_rabbits = 0
def __init__(self, name = 'Rabbit',
fur_colour = 'unknown',
eye_colour = 'unknown',
sex = 'unsexed',
breed = 'unknown',
dob = datetime.now(),
tob = datetime.now()
):
self.name = name # <- Instance variable
self.fur_colour = fur_colour
self.eye_colour = eye_colour
self.sex = sex
self.breed = breed
self.dob = str(dob.strftime("%x"))
self.tob = str(tob.strftime("%X"))
Rabbit_birth.num_of_rabbits += 1
def display_rabbit(self):
dob_time = self.dob+" at "+self.tob
return ("{0.name} is a {0.sex} rabbit"
" and has {0.fur_colour} fur and "
"{0.eye_colour} eyes, {0.name} is "
"of the {0.breed} breed of rabbits.\n"
"{0.name}'s DOB is ").format(self) + dob_time
################################################
print()
print('Number of rabbits is ',Rabbit_birth.num_of_rabbits)
rabbits=[]
def create_rabbit():
lr=[]
rabbit = ('name', 'fur colour', 'eye colour', 'sex', 'breed')
for item in rabbit:
r = input(item+' >')
lr.append(r)
r0,r1,r2,r3,r4=lr
name = Rabbit_birth(r0,r1,r2,r3,r4)
return name
for i in range(3):
name=create_rabbit()
rabbits.append(name)
print(rabbits[i].display_rabbit())
I see the mistake I have made, it's a matter of passing the datetime object in the wrong scope.
Here is the relevant corrected code:
## <- class experiment
from datetime import datetime
class Rabbit_birth:
num_eyes = 2 # <- Class variable
num_legs = 4
num_ears = 2
num_tails = 1
num_of_rabbits = 0
def __init__(self, name = 'Rabbit',
fur_colour = 'unknown',
eye_colour = 'unknown',
sex = 'unsexed',
breed = 'unknown',
dob = None, # This line changed
tob = None # This line changed
):
self.name = name # <- Instance variable
self.fur_colour = fur_colour
self.eye_colour = eye_colour
self.sex = sex
self.breed = breed
self.dob = datetime.now() # This line changed
self.tob = datetime.now() # This line changed
Rabbit_birth.num_of_rabbits += 1
def display_rabbit(self):
# This line below changed
dob_time = str(self.dob.strftime("%x"))+" at "+str(self.tob.strftime("%X"))
return ("{0.name} is a {0.sex} rabbit"
" and has {0.fur_colour} fur and "
"{0.eye_colour} eyes, {0.name} is "
"of the {0.breed} breed of rabbits.\n"
"{0.name}'s DOB is ").format(self) + dob_time
################################################
I've chosen the t2.2xlarge instance with 8 CPUs and 32GiB in memory. However, I feel like the performance is the same as compared to the "free tier" version I used to run my python script on. When I look at the CPU usage on my machine, it says only 8%.
How can I utilize much more of my CPUs?
Here is the following code I'm currently running on these EC2 Instances:
def connectToDB():
databaseServerIP = "mydb.us-east-2.rds.amazonaws.com" # IP address of the MySQL database server
databaseUserName = "mydbUsername" # User name of the database server
databaseUserPassword = "mypwd" # Password for the database user
cursorType = pymysql.cursors.DictCursor
connectionInstance = pymysql.connect(host=databaseServerIP,
user=databaseUserName,
password=databaseUserPassword,
cursorclass=cursorType,
autocommit=True)
# Create a cursor object
cursorInstance = connectionInstance.cursor()
return connectionInstance, cursorInstance
def construct_each_company(tmpDF_forPeerGroup, ii):
print(tmpDF_forPeerGroup['Name'].values[ii])
finalBigDataframe = pd.DataFrame(date_generated, index = date_generated)
#symbolToCheck = tmpDF_forPeerGroup['Symbol'].values[ii]
idx = tmpDF_forPeerGroup.index[ii]
#####################
#####################
##### dataframe 1
try:
connectionInstance, cursorInstance = connectToDB()
sql = "SELECT * FROM DB1.Scores WHERE company_idx = "+str(idx)
finalBigDataframe_1 = pd.read_sql(sql, con=connectionInstance)
except:
finalBigDataframe_1 = None
#####################
#####################
##### dataframe 2
try:
connectionInstance, cursorInstance = connectToDB()
sql = "SELECT * FROM DB2.Scores WHERE company_idx = "+str(idx)
finalBigDataframe_2 = pd.read_sql(sql, con=connectionInstance)
except:
finalBigDataframe_2 = None
#####################
#####################
##### dataframe 3
try:
connectionInstance, cursorInstance = connectToDB()
sql = "SELECT * FROM DB3.Scores WHERE company_idx = "+str(idx)
finalBigDataframe_3 = pd.read_sql(sql, con=connectionInstance)
except:
finalBigDataframe_3 = None
#####################
#####################
##### dataframe 4
try:
connectionInstance, cursorInstance = connectToDB()
sql = "SELECT * FROM DB4.Scores WHERE company_idx = "+str(idx)
finalBigDataframe_4 = pd.read_sql(sql, con=connectionInstance)
except:
finalBigDataframe_4 = None
##################
##################
##################
##################
# merge for every input
# this is not right though...
tmpList_forThisCompany = [finalBigDataframe_1, finalBigDataframe_2, finalBigDataframe_3, finalBigDataframe_4]
return (ii, tmpList_forThisCompany)
def collect_result(result):
global results
results.append(result)
import multiprocessing as mp
for elem_PeerGroup in list(sorted(finalDict))[:]:
print(elem_PeerGroup)
#elem_PeerGroup = 'Africa - Banks'
########################################
### FOR ALL COMPANIES IN THIS PEER GROUP
tmpDF_forPeerGroup = finalDict[elem_PeerGroup]
if len(tmpDF_forPeerGroup)!=0:
########################
## CREATE A FINAL LIST FOR COMPANIES
#finalListForCompanies = []
########################
## CREATE DATETIME RANGE
start = datetime.strptime("01-01-2004", "%d-%m-%Y")
end = datetime.strptime("06-04-2019", "%d-%m-%Y")
date_generated = [start + timedelta(days=x) for x in range(0, (end-start).days)]
# each process will use each CPU
#pool = mp.Pool(mp.cpu_count())
pool = mp.Pool(2)
results=[]
for ii in range(0, len(tmpDF_forPeerGroup)):
pool.apply_async(construct_each_company, args=(tmpDF_forPeerGroup, ii), callback=collect_result)
pool.close()
# postpones the execution of next line of code until all processes in the queue are done.
pool.join()
# Step 5: Sort results [OPTIONAL]
results.sort(key=lambda x: x[0])
finalListForCompanies = [r for (ii, r) in results]
else:
continue
finalScores = []
# for each dataframe, NORMALIZE the companies in the PEER GROUP
for kk in range(4):
#print(kk)
tmpListForNormalisation=[]
for elem in finalListForCompanies:
tmpListForNormalisation.append(elem[kk])
dict_of_dfs = dict(enumerate(tmpListForNormalisation))
try:
dframes = pd.concat(dict_of_dfs)
except:
finalScores.append(None)
continue
dframes = dframes.iloc[:,1:]
if len(dframes)==0:
finalScores.append(None)
continue
if len(dframes)==len(dframes.groupby(level=1)):
arrayTest=[]
for k in range(len(tmpListForNormalisation)):
if (tmpListForNormalisation[k] is None) or (len(tmpListForNormalisation[k])==0):
arrayTest.append(None)
else:
arrayTest.append(tmpListForNormalisation[k])
# put the final result into a list
dict_of_dfs2 = dict(enumerate(arrayTest))
finalScores.append(dict_of_dfs2)
else:
test = dframes.groupby(level=1).pipe(lambda g: dframes.sub(g.mean(), level=1).div(g.std(), level=1))
tmpListForNormalisation2=[]
for date, new_df in test.groupby(level=0):
tmpListForNormalisation2.append(new_df)
arrayTest=[]
j=0
for k in range(len(tmpListForNormalisation)):
if (tmpListForNormalisation[k] is None) or (len(tmpListForNormalisation[k])==0):
arrayTest.append(None)
else:
arrayTest.append(tmpListForNormalisation2[j])
j+=1
test_min = test.min(level=1)
test_max = test.max(level=1)
dict_of_dfs2 = dict(enumerate(arrayTest))
def nrm(d):
_d = d
_d.index = _d.index.get_level_values(1)
NewRange = np.array([0, 100])
o = test_max - test_min
n = NewRange[1] - NewRange[0]
return (((_d - test_min) * n) / o) + NewRange[0]
for k, d in dict_of_dfs2.items():
if d is None:
continue
d.loc[:] = nrm(d).rolling(window=7).mean()
# put the final result into a list
finalScores.append(dict_of_dfs2)
# take the final MEAN for every company
for ll in range(len(tmpDF_forPeerGroup)):
namex = tmpDF_forPeerGroup['Name'].values[ll]
print("Inserting to DB...", namex)
company_idx = tmpDF_forPeerGroup['Company_idx'].values[ll]
company_symbol = tmpDF_forPeerGroup['Symbol'].values[ll]
industryName = tmpDF_forPeerGroup['GICS_Industry_Name'].values[ll]
try:
val1 = finalScores[0][ll]
except:
val1 = None
try:
val2 = finalScores[1][ll]
except:
val2 = None
try:
val3 = finalScores[2][ll]
except:
val3 = None
try:
val4 = finalScores[3][ll]
except:
val4 = None
tmpList = [val1, val2, val3, val4]
tmpDf = dict(enumerate(tmpList))
dframes = pd.concat(tmpDf)
finfin = dframes.mean(level=1)
# adjust according to its industry weights
finfin = adjustWeights(industryName, finfin)
# take data from 01.01.2007 onwards only
finfin = finfin['2007/01/01':]
#####################
# NOW PUT TO DATABASE
engine = create_engine("mysql://mydb.us-east-2.rds.amazonaws.com/"+newDatabaseName)
con = engine.connect()
finfin['timestamp'] = finfin.index
finfin['company_idx'] = [company_idx]*len(finfin)
finfin['company_symbol'] = [company_symbol]*len(finfin)
finfin.to_sql(name='Scores', con=con, if_exists='append', index=False)
I don't see why my VM is only using 8% of my CPU in this case. I don't see any error in my code as it should loop over many different companies and allocate one CPU per company.