I have
print(len(pcb.resources))
for res in pcb.resources:
print ("Releasing resource: " + res.name)
self.releaseResource(res.name, pcb)
print("After: " + str(len(pcb.resources)))
Which outputs
2 # from 1st print: this is correct, I have 2 elements in list
Releasing resource: R1 # is it not looping through the 2nd element? Notice the len() is 2
After: 1 # from print after loop. I am expecting 0
UPDATE
I notice its something to do with the function call. releaseResource. But how might releaseResource be affecting the calling loop?
def releaseResource(self, name, pcb = None):
callScheduler = pcb is not None # if pcb is set, means calling from release resource, so dont call scheduler 1st
if pcb is None:
pcb = self.running
# check if resource exists
if not any(rcb.name == name for rcb in self.resources):
return False
rcb = next(r for r in self.resources if r.name == name)
# remove resource from running pcb's resources
pcb.resources.remove(rcb)
if len(rcb.waitingList) == 0:
# no waiting processes: resource is free
rcb.status = RCB.STATUS_FREE
rcb.heldBy = None
else:
# make dequeue from resource waiting list
pcb = rcb.waitingList.popleft()
# put resource into process resources list
pcb.resources.append(rcb)
rcb.heldBy = pcb
# make it ready
pcb.status = PCB.STATUS_READY
pcb.statusList = self.readyList
self.readyList.enqueue(pcb)
# call scheduler
if callScheduler:
self.scheduler()
return True
It looks to me like you modify the list in releaseResource you iterate over, namely in the line pcb.resources.remove(rcb). Try this instead:
for res in pcb.resources[:]:
print ("Releasing resource: " + res.name)
self.releaseResource(res.name, pcb)
See the documentation.
Related
I want to achieve a function that can fetch data in parallel.
The background is the information of 100 sites can be fetched from site A.
the same account can't be used more than once at a time, so I created 5 different accounts on site A that eanble me to fetch information with 5 accounts.
account info like
worker1 pawd
worker2 pawd
worker3 pawd
worker4 pawd
worker5 pawd
if you want to get information of site B from site A .
then you need to type cmd like get info for siteB_IP on site A.
suppose there are 100 IPs are stored in a list names IPlist
how to fetch information of 100 IPs with 5 avaliable accounts in parallel by threading , and then
all of the information can be sotored in a variable without conflict.
what I have tried is below , below codes can not be executed due to I have no way to achieve the solution:
import threading
user = 'root'
pwd = 'Changeme123'
# the first step is to logon with default account
rs = link.send_cmd(r':lognew:' + '"' + user + '","' + pwd + '"')
# then get all nebor ip from the logon site, the function parse_multi is used for parsing data
IPlist = parse_multi(link.send_cmd('get-IP-info:0xffff'))
def Fetchinfo(user, ip):
rs = link.send_cmd(r':lognew:' + '"' + user + '","' + pwd + '"')
areainfo = link.send_cmd('get info for ' + site_IP)
for ip in IPlist:
# how to handle 100 IPs in the situstion of 5 accounts avaliable ?
thread = threading.thread(target = Fetchinfo, args = [worker, ip]
Since you don't want calls from the same account id and passwords to happen concurrently, you can define a function that sequentially loops through a sub-list of IPs to fetch synchronously:
def fetch_data_for_ips(account_id, account_password, ips_to_fetch):
results = list()
for ip_to_fetch in ips_to_fetch:
# fetch with the account_id and password synchronously
result = ...
results.append(result)
return results // Added this
Then, use a thread pool, to run the different batches concurrently for each account:
from concurrent.futures import ThreadPoolExecutor, as_completed
# Split the workload for each account to fetch
num, remainder = divmod(len(ip_list), len(accounts))
num_ips_for_each_account = num + bool(remainder)
# This gives e.g. [[1,2,3], [4,5,6]], where each sublist is for each account to fetch
ip_lists_for_each_account = [ip_list[i: i + num_ips_for_each_account] for i in range(0, len(ip_list), num_ips_for_each_account)]
# You should only need number of threads = to the number of accounts you have
with ThreadPoolExecutor(len(accounts)) as executor:
# Feel free to use a set instead if you don't need to know which result came from which thread
futures = dict()
results = list()
for (account_id, account_password), ips_to_fetch in zip(accounts, ip_lists_for_each_account):
future = executor.submit(fetch_data_for_ips, account_id, account_password, ips_to_fetch)
futures[future] = account_id
for future in as_completed(futures):
result = future.result()
account_id = futures[future]
print(f'{account_id} fetched these:', result)
results.extend(result)
you can refer to below sample code as rcshon suggested .
def fetch_data_for_ips(account_id,ips_to_fetch):
results = list()
for ip_to_fetch in ips_to_fetch:
# fetch with the account_id and password synchronously
result = ','.join((account_id,ip_to_fetch))
results.append(result)
return results
from concurrent.futures import ThreadPoolExecutor, as_completed
accounts = ['worker1','worker2','worker3','worker4','worker5']
ip_list = [str(_) for _ in range(10)]
# Split the workload for each account to fetch
num, remainder = divmod(len(ip_list), len(accounts))
num_ips_for_each_account = num + bool(remainder)
# This gives e.g. [[1,2,3], [4,5,6]], where each sublist is for each account to fetch
ip_lists_for_each_account = [ip_list[i: i + num_ips_for_each_account] for i in range(0, len(ip_list), num_ips_for_each_account)]
# You should only need number of threads = to the number of accounts you have
with ThreadPoolExecutor(len(accounts)) as executor:
# Feel free to use a set instead if you don't need to know which result came from which thread
futures = dict()
results = list()
for account_id, ips_to_fetch in zip(accounts, ip_lists_for_each_account):
future = executor.submit(fetch_data_for_ips, account_id, ips_to_fetch)
futures[future] = account_id
for future in as_completed(futures):
result = future.result()
account_id = futures[future]
print(f'{account_id} fetched these:', result)
results.extend(result)
output :
worker3 fetched these: ['worker3,4', 'worker3,5']
worker2 fetched these: ['worker2,2', 'worker2,3']
worker1 fetched these: ['worker1,0', 'worker1,1']
worker4 fetched these: ['worker4,6', 'worker4,7']
worker5 fetched these: ['worker5,8', 'worker5,9']
I am trying to make resources unavailable for a certain time in simpy. The issue is with timeout I find the resource is still active and serving during the time it should be unavailable. Can anyone help me with this in case you have encountered such a problem. Thanks a lot!
import numpy as np
import simpy
def interarrival():
return(np.random.exponential(10))
def servicetime():
return(np.random.exponential(20))
def servicing(env, servers_1):
i = 0
while(True):
i = i+1
yield env.timeout(interarrival())
print("Customer "+str(i)+ " arrived in the process at "+str(env.now))
state = 0
env.process(items(env, i, servers_array, state))
def items(env, customer_id, servers_array, state):
with servers_array[state].request() as request:
yield request
t_arrival = env.now
print("Customer "+str(customer_id)+ " arrived in "+str(state)+ " at "+str(t_arrival))
yield env.timeout(servicetime())
t_depart = env.now
print("Customer "+str(customer_id)+ " departed from "+str(state)+ " at "+str(t_depart))
if (state == 1):
print("Customer exists")
else:
state = 1
env.process(items(env, customer_id, servers_array, state))
def delay(env, servers_array):
while(True):
if (env.now%1440 >= 540 and env.now <= 1080):
yield(1080 - env.now%1440)
else:
print(str(env.now), "resources will be blocked")
resource_unavailability_dict = dict()
resource_unavailability_dict[0] = []
resource_unavailability_dict[1] = []
for nodes in resource_unavailability_dict:
for _ in range(servers_array[nodes].capacity):
resource_unavailability_dict[nodes].append(servers_array[nodes].request())
print(resource_unavailability_dict)
for nodes in resource_unavailability_dict:
yield env.all_of(resource_unavailability_dict[nodes])
if (env.now < 540):
yield env.timeout(540)
else:
yield env.timeout((int(env.now/1440)+1)*1440+540 - env.now)
for nodes in resource_unavailability_dict:
for request in resource_unavailability_dict[nodes]:
servers_array[nodes].release(request)
print(str(env.now), "resources are released")
env = simpy.Environment()
servers_array = []
servers_array.append(simpy.Resource(env, capacity = 5))
servers_array.append(simpy.Resource(env, capacity = 7))
env.process(servicing(env, servers_array))
env.process(delay(env,servers_array))
env.run(until=2880)
The code is given above. Actually, I have two nodes 0 and 1 where server capacities are 5 and 7 respectively. The servers are unavailable before 9AM (540 mins from midnight) and after 6 PM everyday. I am trying to create the unavailability using timeout but not working. Can you suggest how do I modify the code to incorporate it.
I am getting the error AttributeError: 'int' object has no attribute 'callbacks'which I can't figure out why ?
So the problem with simpy resources is the capacity is a read only attribute. To get around this you need something to seize and hold the resource off line. So in essence, I have two types of users, the ones that do "real work" and the ones that control the capacity. I am using a simple resource, which means that the queue at the schedule time will get processed before the capacity change occurs. Using a priority resource means the current users of a resource can finish their processes before the capacity change occurs , or you can use a pre-emptive resource to interrupt users with resources at the scheduled time. here is my code
"""
one way to change a resouce capacity on a schedule
note the the capacity of a resource is a read only atribute
Programmer: Michael R. Gibbs
"""
import simpy
import random
def schedRes(env, res):
"""
Performs maintenance at time 100 and 200
waits till all the resources have been seized
and then spend 25 time units doing maintenace
and then release
since I am using a simple resource, maintenance
will wait of all request that are already in
the queue when maintenace starts to finish
you can change this behavior with a priority resource
or pre-emptive resource
"""
# wait till first scheduled maintenance
yield env.timeout(100)
# build a list of requests for each resource
# then wait till all requests are filled
res_maint_list = []
print(env.now, "Starting maintenance")
for _ in range(res.capacity):
res_maint_list.append(res.request())
yield env.all_of(res_maint_list)
print(env.now, "All resources seized for maintenance")
# do maintenance
yield env.timeout(25)
print(env.now, "Maintenance fisish")
# release all the resources
for req in res_maint_list:
res.release(req)
print(env.now,"All resources released from maint")
# wait till next scheduled maintenance
dur_to_next_maint = 200 -env.now
if dur_to_next_maint > 0:
yield env.timeout(dur_to_next_maint)
# do it all again
res_maint_list = []
print(env.now, "Starting maintenance")
for _ in range(res.capacity):
res_maint_list.append(res.request())
yield env.all_of(res_maint_list)
print(env.now, "All resources seized for maintenance")
yield env.timeout(25)
print(env.now, "Maintenance fisish")
for req in res_maint_list:
res.release(req)
print(env.now,"All resources released from maint")
def use(env, res, dur):
"""
Simple process of a user seizing a resource
and keeping it for a little while
"""
with res.request() as req:
print(env.now, f"User is in queue of size {len(res.queue)}")
yield req
print(env.now, "User has seized a resource")
yield env.timeout(dur)
print(env.now, "User has released a resource")
def genUsers(env,res):
"""
generate users to seize resources
"""
while True:
yield env.timeout(10)
env.process(use(env,res,21))
# set up
env = simpy.Environment()
res = simpy.Resource(env,capacity=2) # may want to use a priority or preemtive resource
env.process(genUsers(env,res))
env.process(schedRes(env, res))
# start
env.run(300)
One way to do this is with preemptive resources. When it is time to make resources unavailable, issue a bunch of requests with the highest priority to seize idle resources, and to preempt resources currently in use. These requests would then release the resources when its time to make the resources available again. Note that you will need to add some logic on how the preempted processes resume once the resources become available again. If you do not need to preempt processes, you can just use priority resources instead of preemptive resources
I am writing tests for an API with pytest.
The tests are structured like that:
KEEP_BOX_IDS = ["123abc"]
#pytest.fixture(scope="module")
def s():
UID = os.environ.get("MYAPI_UID")
if UID is None:
raise KeyError("UID not set in environment variable")
PWD = os.environ.get("MYAPI_PWD")
if PWD is None:
raise KeyError("PWD not set in environment variable")
return myapi.Session(UID, PWD)
#pytest.mark.parametrize("name,description,count", [
("Normal Box", "Normal Box Description", 1),
("ÄäÖöÜüß!§", "ÄäÖöÜüß!§", 2),
("___--_?'*#", "\n\n1738\n\n", 3),
])
def test_create_boxes(s, name, description, count):
box_info_create = s.create_box(name, description)
assert box_info_create["name"] == name
assert box_info_create["desc"] == description
box_info = s.get_box_info(box_info_create["id"])
assert box_info["name"] == name
assert box_info["desc"] == description
assert len(s.get_box_list()) == count + len(KEEP_BOX_IDS)
def test_update_boxes(s):
bl = s.get_box_list()
for b in bl:
b_id = b['id']
if b_id not in KEEP_BOX_IDS:
new_name = b["name"] + "_updated"
new_desc = b["desc"] + "_updated"
s.update_box(b_id, new_name, new_desc)
box_info = s.get_box_info(b_id)
assert box_info["name"] == new_name
assert get_box_info["desc"] == new_desc
I use a fixture to set up the session (this will keep me connected to the API).
As you can see I am creating 3 boxes at the beginning.
All test that are following do some sort of operations on this 3 boxes. (Boxes are just spaces for folders and files)
For example: update_boxes, create_folders, rename_folders, upload_files, change_file names, etc..
I know it's not good, since all the tests are dependent from each other, but if I execute them in the right order the test is valid and thats enough.
The second issue, which borders me the most, is that all the following tests start with the same lines:
bl = s.get_box_list()
for b in bl:
b_id = b['id']
if b_id not in KEEP_BOX_IDS:
box_info = s.get_box_info(b_id)
I always need to call this for loop to get each boxs id and info.
I've tried to put it in a second fixture, but the problem is that then there will be two fixtures.
Is there a better way of doing this?
Thanks
New to Python/Boto3, this should be an easy one but still learning :)
I have a Lambda function which creates a number of snapshots and works fine:
def create_snapshot():
volumes = ec2_client.describe_volumes(
Filters=[
{'N'...
...
for volume in volumes...
....
snap_name = 'Backup of ' + snap_desc
....
snap = ec2_client.create_snapshot(
VolumeId=vol_id,
Description=snap_desc
)
I then want to receive an email from AWS SNS to let me know which snapshots the function created, which I do using:
message = sns.publish(
TopicArn=SNSARN,
Subject=("Function executed"),
Message=("%s created" % snap_name)
)
The issue is that this creates an email for each snapshot, instead of one email listing all the snapshots. Should I create another function that calls all values produced by snap_desc, or can I send all values for snap_desc in the function? And most importantly what's the best way of doing this?
Cheers!
Scott
####################### UPDATE (Thanks #omuthu) #######################
I set an array inside and outside the loop, and put the string into the message. This produced the following being sent in one message:
The following snapshots have been created:
['vol-0e0b9a5dfb8379fc0 (Instance 1 - /dev/sda1)', 'vol-03aac6b65df64661e (Instance 4 - /dev/sda1)', 'vol-0fdde765dfg452631 (Instance 2 - /dev/sda1)', 'vol-0693a9568b11f625f (Instance 3 - /dev/sda1)', etc.
Okay got it sorted, finally!
def create_snapshot():
volumes = ec2_client.describe_volumes(
Filters=[
{'N'...
...
inst_list = []
for volume in volumes...
vol_id = volume['VolumeId']
....
snap_desc = vol_id
for name in volume['Tags']:
tag_key = name['Key']
tag_val = name['Value']
if tag_key == 'Name':
snap_desc = vol_id + ' (' + tag_val + ')'
....
....
....
if backup_mod is False or (current_hour + 10) % backup_mod != 0:
...
continue
else:
print("%s is scheduled this hour" % vol_id)
for name in volume['Tags']:
inst_tag_key = name['Key']
inst_tag_val = name['Value']
if inst_tag_key == 'Name':
inst_list.append(inst_tag_val)
snap = ec2_client.create_snapshot(
VolumeId=vol_id,
Description=snap_desc,
)
print("%s created" % snap['SnapshotId'])
msg = str("\n".join(inst_list))
if len(inst_list) != 0:
message = sns.publish(
TopicArn=SNSARN,
Subject=("Daily Lambda snapshot function complete"),
Message=("The following snapshots have been created today:\n\n" + msg + "\n")
)
print("Response: {}".format(message))
I'm not sure if the title accurately describes what I'm trying to do. I have a Python3.x script that I wrote that will issue flood warning to my facebook page when the river near my home has reached it's lowest flood stage. Right now the script works, however it only reports data from one measuring station. I would like to be able to process the data from all of the stations in my county (total of 5), so I was thinking that maybe a class method may do the trick but I'm not sure how to implement it. I've been teaching myself Python since January and feel pretty comfortable with the language for the most part, and while I have a good idea of how to build a class object I'm not sure how my flow chart should look. Here is the code now:
#!/usr/bin/env python3
'''
Facebook Flood Warning Alert System - this script will post a notification to
to Facebook whenever the Sabine River # Hawkins reaches flood stage (22.3')
'''
import requests
import facebook
from lxml import html
graph = facebook.GraphAPI(access_token='My_Access_Token')
river_url = 'http://water.weather.gov/ahps2/river.php?wfo=SHV&wfoid=18715&riverid=203413&pt%5B%5D=147710&allpoints=143204%2C147710%2C141425%2C144668%2C141750%2C141658%2C141942%2C143491%2C144810%2C143165%2C145368&data%5B%5D=obs'
ref_url = 'http://water.weather.gov/ahps2/river.php?wfo=SHV&wfoid=18715&riverid=203413&pt%5B%5D=147710&allpoints=143204%2C147710%2C141425%2C144668%2C141750%2C141658%2C141942%2C143491%2C144810%2C143165%2C145368&data%5B%5D=all'
def checkflood():
r = requests.get(river_url)
tree = html.fromstring(r.content)
stage = ''.join(tree.xpath('//div[#class="stage_stage_flow"]//text()'))
warn = ''.join(tree.xpath('//div[#class="current_warns_statmnts_ads"]/text()'))
stage_l = stage.split()
level = float(stage_l[2])
#check if we're at flood level
if level < 22.5:
pass
elif level == 37:
major_diff = level - 23.0
major_r = ('The Sabine River near Hawkins, Tx has reached [Major Flood Stage]: #', stage_l[2], 'Ft. ', str(round(major_diff, 2)), ' Ft. \n Please click the link for more information.\n\n Current Warnings and Alerts:\n ', warn)
major_p = ''.join(major_r)
graph.put_object(parent_object='me', connection_name='feed', message = major_p, link = ref_url)
<--snip-->
checkflood()
Each station has different 5 different catagories for flood stage: Action, Flood, Moderate, Major, each different depths per station. So for Sabine river in Hawkins it will be Action - 22', Flood - 24', Moderate - 28', Major - 32'. For the other statinos those depths are different. So I know that I'll have to start out with something like:
class River:
def __init__(self, id, stage):
self.id = id #station ID
self.stage = stage #river level'
#staticmethod
def check_flood(stage):
if stage < 22.5:
pass
elif stage.....
but from there I'm not sure what to do. Where should it be added in(to?) the code, should I write a class to handle the Facebook postings as well, is this even something that needs a class method to handle, is there any way to clean this up for efficiency? I'm not looking for anyone to write this up for me, but some tips and pointers would sure be helpful. Thanks everyone!
EDIT Here is what I figured out and is working:
class River:
name = ""
stage = ""
action = ""
flood = ""
mod = ""
major = ""
warn = ""
def checkflood(self):
if float(self.stage) < float(self.action):
pass
elif float(self.stage) >= float(self.major):
<--snip-->
mineola = River()
mineola.name = stations[0]
mineola.stage = stages[0]
mineola.action = "13.5"
mineola.flood = "14.0"
mineola.mod = "18.0"
mineola.major = "21.0"
mineola.alert = warn[0]
hawkins = River()
hawkins.name = stations[1]
hawkins.stage = stages[1]
hawkins.action = "22.5"
hawkins.flood = "23.0"
hawkins.mod = "32.0"
hawkins.major = "37.0"
hawkins.alert = warn[1]
<--snip-->
So from here I'm tring to stick all the individual river blocks into one block. What I have tried so far is this:
class River:
... name = ""
... stage = ""
... def testcheck(self):
... return self.name, self.stage
...
>>> for n in range(num_river):
... stations[n] = River()
... stations[n].name = stations[n]
... stations[n].stage = stages[n]
...
>>> for n in range(num_river):
... stations[n].testcheck()
...
<__main__.River object at 0x7fbea469bc50> 4.13
<__main__.River object at 0x7fbea46b4748> 20.76
<__main__.River object at 0x7fbea46b4320> 22.13
<__main__.River object at 0x7fbea46b4898> 16.08
So this doesn't give me the printed results that I was expecting. How can I return the string instead of the object? Will I be able to define the Class variables in this manner or will I have to list them out individually? Thanks again!
After reading many, many, many articles and tutorials on class objects I was able to come up with a solution for creating the objects using list elements.
class River():
def __init__(self, river, stage, flood, action):
self.river = river
self.stage = stage
self.action = action
self.flood = flood
self.action = action
def alerts(self):
if float(self.stage < self.flood):
#alert = "The %s is below Flood Stage (%sFt) # %s Ft. \n" % (self.river, self.flood, self.stage)
pass
elif float(self.stage > self.flood):
alert = "The %s has reached Flood Stage(%sFt) # %sFt. Warnings: %s \n" % (self.river, self.flood, self.stage, self.action)
return alert
'''this is the function that I was trying to create
to build the class objects automagically'''
def riverlist():
river_list = []
for n in range(len(rivers)):
station = River(river[n], stages[n], floods[n], warns[n])
river_list.append(station)
return river_list
if __name__ == '__main__':
for x in riverlist():
print(x.alerts())