Related
Hi is there any method for apply trasnformation for certain batch?
It means, I want apply trasnformation for just last batch in every epochs.
What I tried is here
import torch
class test(torch.utils.data.Dataset):
def __init__(self):
self.source = [i for i in range(10)]
def __len__(self):
return len(self.source)
def __getitem__(self, idx):
print(idx)
return self.source[idx]
ds = test()
dl = torch.utils.data.DataLoader(dataset = ds, batch_size = 3,
shuffle = False, num_workers = 5)
for i in dl:
print(i)
because I thought that if I could get idx number, it would be possible to apply for certain batchs.
However If using num_workers outputs are
0
1
2
3
964
57
8
tensor([0, 1, 2])
tensor([3, 4, 5])
tensor([6, 7, 8])
tensor([9])
which are not I thought
without num_worker
0
1
2
tensor([0, 1, 2])
3
4
5
tensor([3, 4, 5])
6
7
8
tensor([6, 7, 8])
9
tensor([9])
So the question is
Why idx works so with num_workers?
How can I apply trasnform for certain batchs (or certain idx)?
When you have num_workers > 1, you have multiple subprocesses doing data loading in parallel. So what is likely happening is that there is a race condition for the print step, and the order you see in the output depends on which subprocess goes first each time.
For most transforms, you can apply them on a specific batch simply by calling the transform after the batch has been loaded. To do this just for the last batch, you could do something like:
for batch_idx, batch_data in dl:
# check if batch is the last batch
if ((batch_idx+1) * batch_size) >= len(ds):
batch_data = transform(batch_data)
I found that
class test_dataset(torch.utils.data.Dataset):
def __init__(self):
self.a = [i for i in range(100)]
def __len__(self):
return len(self.a)
def __getitem__(self, idx):
a = torch.tensor(self.a[idx])
#print(idx)
return idx
a = torch.utils.data.DataLoader(
test_dataset(), batch_size = 10, shuffle = False,
num_workers = 10, pin_memory = True)
for i in a:
print(i)
tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
tensor([10, 11, 12, 13, 14, 15, 16, 17, 18, 19])
tensor([20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
tensor([30, 31, 32, 33, 34, 35, 36, 37, 38, 39])
tensor([40, 41, 42, 43, 44, 45, 46, 47, 48, 49])
tensor([50, 51, 52, 53, 54, 55, 56, 57, 58, 59])
tensor([60, 61, 62, 63, 64, 65, 66, 67, 68, 69])
tensor([70, 71, 72, 73, 74, 75, 76, 77, 78, 79])
tensor([80, 81, 82, 83, 84, 85, 86, 87, 88, 89])
tensor([90, 91, 92, 93, 94, 95, 96, 97, 98, 99])
Read input as specified in the question
Print output as specified in the question.
arr=[34, 57, 82, 41, 65, 35, 82, 27, 36, 12, 6, 40, 66, 99, 25, 29, 22, 25, 12, 24, 65, 15, 5, 43, 28, 33, 76, 32, 13, 95, 22, 84, 71, 23, 28, 7, 65, 94, 18, 47, 9, 42, 61, 73]
x=61
si = 0
def lastIndex(arr, x, si):
arrLen = len(arr)
lastKnownIndex = -1
if (arrLen == 0):
return lastKnownIndex
if (si == arrLen):
if (arr[si] == x):
lastKnownIndex = si
return lastKnownIndex
if (arr[si] == x):
lastKnownIndex = si
lastIndex(arr, x, si + 1)
return lastKnownIndex
print(lastIndex(arr, x, si))
You do not have an infinite recursion but an IndexError exception. The problem comes from these two lines:
if (si == arrLen):
if (arr[si] == x):
The second line will always raise an exception since if si == arrLen then arr[si] is equivalent to arr[len(arr)] which is always wrong (remember that list items are indexed from 0 to len(arr) - 1).
Here is a patched version that fixes this bug and simplifies a bit your code:
arr=[34, 57, 82, 41, 65, 35, 82, 27, 36, 12, 6, 40, 66, 99, 25, 29, 22, 25, 12, 24, 65, 15, 5, 43, 28, 33, 76, 32, 13, 95, 22, 84, 71, 23, 28, 7, 65, 94, 18, 47, 9, 42, 61, 73]
def lastIndex(arr, x, si=0):
# empty list or end of list reached
if si >= len(arr):
return -1
# x is at position si
if arr[si] == x:
return si
# owtherwise look at next position
return lastIndex(arr, x, si + 1)
print(lastIndex(arr, 61)) # print 42
I am trying to generate 2 list with different size consisting with random numbers. I can generate 2 list with random numbers, but how to achieve 2 different length of lists?
import random
list1 = random.sample(xrange(100), 10)
list2 = random.sample(xrange(100), 10)
print(list1)
print(list2)
Need to generate the lists with 2 random different sizes as well, as if both the lists are completely random.
Try the below code. Hope this would help.
If you want to create random number list of two different sizes. Then you can explicitly, pass the size of the list as a second argument, as given below.
import random
list1 = random.sample(xrange(100), 100)
list2 = random.sample(xrange(100), 10)
print(list1)
print(list2)
Ouput will be :
[46, 73, 13, 89, 44, 23, 74, 8, 19, 79, 36, 80, 85, 42, 82, 39, 61, 15, 27, 68, 67, 30, 11, 21, 86, 16, 63, 95, 17, 90, 37, 81, 20, 71, 93, 99, 40, 6, 47, 92, 58, 35, 12, 2, 10, 98, 87, 50, 51, 97, 70, 65, 78, 22, 72, 45, 59, 0, 52, 14, 1, 84, 43, 24, 54, 31, 18, 69, 7, 75, 53, 25, 57, 94, 83, 66, 3, 5, 88, 32, 4, 28, 29, 55, 9, 77, 60, 62, 41, 76, 48, 56, 34, 91, 33, 96, 49, 38, 26, 64]
[82, 58, 74, 61, 21, 77, 53, 35, 44, 59]
Now if you want to randomly decide the size of the list, the pass a random number as a second argument, by using randint function
import random
list1 = random.sample(range(100), random.randint(1,101))
list2 = random.sample(range(100), random.randint(1,101))
print(list1)
print(list2)
Output would be:
[93, 60, 82, 53, 16, 42, 0, 68, 88, 11, 89, 62, 38, 14, 27, 8, 45, 25, 83, 97, 94]
[30, 5, 19, 11, 14, 6, 7, 86, 16, 53, 71, 12, 90, 32]
You can try something like this, which would randomly generate the size between 1 and 10.
import random
list1 = random.sample(range(100), random.randint(1,10))
list2 = random.sample(range(100), random.randint(1,10))
print(list1)
print(list2)
This will generate random length of the lists. Hope it helps !
You need to randomize the second Parameter as well to become lists of random size:
import random
list1 = random.sample(range(100), random.randint(1,10))
list2 = random.sample(range(100), random.randint(1,10))
print(list1)
print(list2)
In this code used detectors of dlib.
dlib.get_frontal_face_detector()
dlib.cnn_face_detection_model_v1('mmod_human_face_detector.dat')
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('res/model.dat')
# detector = dlib.cnn_face_detection_model_v1('mmod_human_face_detector.dat')
cap = cv.VideoCapture(0)
while True:
_, frame = cap.read(0)
gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
dets = detector(gray, 0)
print(dets)
for det in dets:
landmarks = shape_to_np(predictor(gray, det))
cv.imshow('test', frame)
if cv.waitKey(1) == ord('q'):
break
When used the cnn detector, dets look as:
mmod_rectangles[[(258, 254) (422, 417)]]
And an exception is thrown in the predictor line:
TypeError: __call__(): incompatible function arguments. The following
argument types are supported:
1. (self: dlib.shape_predictor, image: array, box: dlib.rectangle) -> dlib.full_object_detection
Invoked with: <dlib.shape_predictor object at 0x7f37a12ba9d0>,
array([[71, 69, 70, ..., 71, 70, 73],
[71, 72, 71, ..., 72, 72, 75],
[71, 70, 71, ..., 72, 72, 73],
...,
[27, 27, 27, ..., 75, 71, 68],
[27, 27, 27, ..., 74, 71, 71],
[24, 25, 27, ..., 73, 71, 70]], dtype=uint8), <dlib.mmod_rectangle object at 0x7f37819beea0>
But when used get_frontal_face_detector, the dets look as:
rectangles[[(273, 234) (453, 413)]]
And the code works correctly.
try to perform
faceRect = det.rect
landmarks = shape_to_np(predictor(gray, faceRect))
perhas version problem
faceRect = det[0].rect
landmarks = shape_to_np(predictor(gray, faceRect))
interface port-channel 1
ALLOWED_VLAN 2,4-7,27,30-31,38-39,41-42,48-50
ALLOWED_VLAN 74,678,1101-1102,1201-1202
interface port-channel 2
ALLOWED_VLAN 37,51-73,75-76,1051-1052,2001
interface port-channel 101
ALLOWED_VLAN 10,18-19,37,39,51-52,75-76,901-902
ALLOWED_VLAN 2901-2902,3204,3305
import re
import itertools
fileOpen3 = open('C:\\Python36\\execrice\\inter.txt')
list3 = []
for line in fileOpen3.readlines():
if line.startswith('ALLOWED_VLAN'):
allowedVlan = re.compile(r'\d+\S+')
list1 = allowedVlan.findall(line)
st1 = list1[0]
pv1 = st1.split(',')
list3.append(pv1)
merged = list(itertools.chain.from_iterable(list3))
singleVlanDigit = []
expandedVlan1 = []
for i in merged:
rangeOfVlan = []
if '-' in i:
rangeOfVlan.append(i)
else:
singleVlanDigit.append(i)
singleVlanDigit = list(map(int,singleVlanDigit))
for j in rangeOfVlan:
l = j.split('-')
startVlan = int(l[0])
endVlan = int(l[1])
for k in range(startVlan,endVlan):
expandedVlan1.append(k)
vlanallowed = singleVlanDigit + expandedVlan1
vlanallowed = list(map(int,vlanallowed))
print (vlanallowed)
elif line.startswith('interface port-channel'):
list3=[]
print ("interface port-channel")
fileOpen3.close()
my program combines all the digits in one single list where as i want it to stop when it reads the "interface port-channel 2" and so on
i want the output of this program to be as below
interface port-channel
[2, 27, 4, 5, 6, 30, 38, 41, 48, 49, 74, 678, 1101, 1201]
interface port-channel
[37, 2001, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 75, 1051]
interface port-channel
[10, 37, 39, 18, 51, 75, 901, 3204, 3305, 2901]
but it produces the below output
interface port-channel
[2, 27, 4, 5, 6, 30, 38, 41, 48, 49]
[2, 27, 74, 678, 4, 5, 6, 30, 38, 41, 48, 49, 1101, 1201]
interface port-channel
[2, 27, 74, 678, 37, 2001, 4, 5, 6, 30, 38, 41, 48, 49, 1101, 1201, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 75, 1051]
interface port-channel
[2, 27, 74, 678, 37, 2001, 10, 37, 39, 4, 5, 6, 30, 38, 41, 48, 49, 1101, 1201, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 75, 1051, 18, 51, 75, 901]
[2, 27, 74, 678, 37, 2001, 10, 37, 39, 3204, 3305, 4, 5, 6, 30, 38, 41, 48, 49, 1101, 1201, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 75, 1051, 18, 51, 75, 901, 2901]
after reinitialising the list3 in elif block as suggested by Boar i got the below results which is very to my end results
interface port-channel
[2, 27, 4, 5, 6, 30, 38, 41, 48, 49]
[2, 27, 74, 678, 4, 5, 6, 30, 38, 41, 48, 49, 1101, 1201]
interface port-channel
[37, 2001, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 75, 1051]
interface port-channel
[10, 37, 39, 18, 51, 75, 901]
[10, 37, 39, 3204, 3305, 18, 51, 75, 901, 2901]
but i want the results to be like this
interface port-channel
[2, 27, 74, 678, 4, 5, 6, 30, 38, 41, 48, 49, 1101, 1201]
interface port-channel
[37, 2001, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 75, 1051]
interface port-channel
[10, 37, 39, 3204, 3305, 18, 51, 75, 901, 2901]
I've had to guess at where the line breaks fall in your data but I think I have it right.
The repeated data is because you are initializing list3 in the wrong place. It should be after if line.startswith('ALLOWED_VLAN'):
With that fix, your program does this:
interface port-channel
[2, 27, 4, 5, 6, 30, 38, 41, 48, 49]
[74, 678, 1101, 1201]
interface port-channel
[37, 2001, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 75, 1051]
interface port-channel
[10, 37, 39, 18, 51, 75, 901]
[3204, 3305, 2901]
which is close to what you want. You just need to combine the lists of integers before printing them. Combine the values of vnlanallowed in the if part of the loop, then print it out and reinitialize it in the elif part.
To get the exact format of the output you want, you need to wait until you know you have all of the lists of port-channel numbers before printing them. You only know that when you see a new list start, or you reach end-of-file.
import re
import itertools
fileOpen3 = open(r'E:\...\inter.txt')
channel_list = []
for line in fileOpen3.readlines():
if line.startswith('ALLOWED_VLAN'):
list3 = []
allowedVlan = re.compile(r'\d+\S+')
list1 = allowedVlan.findall(line)
st1 = list1[0]
pv1 = st1.split(',')
list3.append(pv1)
merged = list(itertools.chain.from_iterable(list3))
singleVlanDigit = []
expandedVlan1 = []
for i in merged:
rangeOfVlan = []
if '-' in i:
rangeOfVlan.append(i)
else:
singleVlanDigit.append(i)
singleVlanDigit = list(map(int,singleVlanDigit))
for j in rangeOfVlan:
l = j.split('-')
startVlan = int(l[0])
endVlan = int(l[1])
for k in range(startVlan,endVlan):
expandedVlan1.append(k)
vlanallowed = singleVlanDigit + expandedVlan1
vlanallowed = list(map(int,vlanallowed))
#print (vlanallowed)
channel_list.extend(vlanallowed)
elif line.startswith('interface port-channel'):
if channel_list:
print ("interface port-channel", channel_list)
channel_list = []
print("interface port-channel", channel_list)
fileOpen3.close()