Hi is there any method for apply trasnformation for certain batch?
It means, I want apply trasnformation for just last batch in every epochs.
What I tried is here
import torch
class test(torch.utils.data.Dataset):
def __init__(self):
self.source = [i for i in range(10)]
def __len__(self):
return len(self.source)
def __getitem__(self, idx):
print(idx)
return self.source[idx]
ds = test()
dl = torch.utils.data.DataLoader(dataset = ds, batch_size = 3,
shuffle = False, num_workers = 5)
for i in dl:
print(i)
because I thought that if I could get idx number, it would be possible to apply for certain batchs.
However If using num_workers outputs are
0
1
2
3
964
57
8
tensor([0, 1, 2])
tensor([3, 4, 5])
tensor([6, 7, 8])
tensor([9])
which are not I thought
without num_worker
0
1
2
tensor([0, 1, 2])
3
4
5
tensor([3, 4, 5])
6
7
8
tensor([6, 7, 8])
9
tensor([9])
So the question is
Why idx works so with num_workers?
How can I apply trasnform for certain batchs (or certain idx)?
When you have num_workers > 1, you have multiple subprocesses doing data loading in parallel. So what is likely happening is that there is a race condition for the print step, and the order you see in the output depends on which subprocess goes first each time.
For most transforms, you can apply them on a specific batch simply by calling the transform after the batch has been loaded. To do this just for the last batch, you could do something like:
for batch_idx, batch_data in dl:
# check if batch is the last batch
if ((batch_idx+1) * batch_size) >= len(ds):
batch_data = transform(batch_data)
I found that
class test_dataset(torch.utils.data.Dataset):
def __init__(self):
self.a = [i for i in range(100)]
def __len__(self):
return len(self.a)
def __getitem__(self, idx):
a = torch.tensor(self.a[idx])
#print(idx)
return idx
a = torch.utils.data.DataLoader(
test_dataset(), batch_size = 10, shuffle = False,
num_workers = 10, pin_memory = True)
for i in a:
print(i)
tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
tensor([10, 11, 12, 13, 14, 15, 16, 17, 18, 19])
tensor([20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
tensor([30, 31, 32, 33, 34, 35, 36, 37, 38, 39])
tensor([40, 41, 42, 43, 44, 45, 46, 47, 48, 49])
tensor([50, 51, 52, 53, 54, 55, 56, 57, 58, 59])
tensor([60, 61, 62, 63, 64, 65, 66, 67, 68, 69])
tensor([70, 71, 72, 73, 74, 75, 76, 77, 78, 79])
tensor([80, 81, 82, 83, 84, 85, 86, 87, 88, 89])
tensor([90, 91, 92, 93, 94, 95, 96, 97, 98, 99])
Related
Read input as specified in the question
Print output as specified in the question.
arr=[34, 57, 82, 41, 65, 35, 82, 27, 36, 12, 6, 40, 66, 99, 25, 29, 22, 25, 12, 24, 65, 15, 5, 43, 28, 33, 76, 32, 13, 95, 22, 84, 71, 23, 28, 7, 65, 94, 18, 47, 9, 42, 61, 73]
x=61
si = 0
def lastIndex(arr, x, si):
arrLen = len(arr)
lastKnownIndex = -1
if (arrLen == 0):
return lastKnownIndex
if (si == arrLen):
if (arr[si] == x):
lastKnownIndex = si
return lastKnownIndex
if (arr[si] == x):
lastKnownIndex = si
lastIndex(arr, x, si + 1)
return lastKnownIndex
print(lastIndex(arr, x, si))
You do not have an infinite recursion but an IndexError exception. The problem comes from these two lines:
if (si == arrLen):
if (arr[si] == x):
The second line will always raise an exception since if si == arrLen then arr[si] is equivalent to arr[len(arr)] which is always wrong (remember that list items are indexed from 0 to len(arr) - 1).
Here is a patched version that fixes this bug and simplifies a bit your code:
arr=[34, 57, 82, 41, 65, 35, 82, 27, 36, 12, 6, 40, 66, 99, 25, 29, 22, 25, 12, 24, 65, 15, 5, 43, 28, 33, 76, 32, 13, 95, 22, 84, 71, 23, 28, 7, 65, 94, 18, 47, 9, 42, 61, 73]
def lastIndex(arr, x, si=0):
# empty list or end of list reached
if si >= len(arr):
return -1
# x is at position si
if arr[si] == x:
return si
# owtherwise look at next position
return lastIndex(arr, x, si + 1)
print(lastIndex(arr, 61)) # print 42
I am trying to generate 2 list with different size consisting with random numbers. I can generate 2 list with random numbers, but how to achieve 2 different length of lists?
import random
list1 = random.sample(xrange(100), 10)
list2 = random.sample(xrange(100), 10)
print(list1)
print(list2)
Need to generate the lists with 2 random different sizes as well, as if both the lists are completely random.
Try the below code. Hope this would help.
If you want to create random number list of two different sizes. Then you can explicitly, pass the size of the list as a second argument, as given below.
import random
list1 = random.sample(xrange(100), 100)
list2 = random.sample(xrange(100), 10)
print(list1)
print(list2)
Ouput will be :
[46, 73, 13, 89, 44, 23, 74, 8, 19, 79, 36, 80, 85, 42, 82, 39, 61, 15, 27, 68, 67, 30, 11, 21, 86, 16, 63, 95, 17, 90, 37, 81, 20, 71, 93, 99, 40, 6, 47, 92, 58, 35, 12, 2, 10, 98, 87, 50, 51, 97, 70, 65, 78, 22, 72, 45, 59, 0, 52, 14, 1, 84, 43, 24, 54, 31, 18, 69, 7, 75, 53, 25, 57, 94, 83, 66, 3, 5, 88, 32, 4, 28, 29, 55, 9, 77, 60, 62, 41, 76, 48, 56, 34, 91, 33, 96, 49, 38, 26, 64]
[82, 58, 74, 61, 21, 77, 53, 35, 44, 59]
Now if you want to randomly decide the size of the list, the pass a random number as a second argument, by using randint function
import random
list1 = random.sample(range(100), random.randint(1,101))
list2 = random.sample(range(100), random.randint(1,101))
print(list1)
print(list2)
Output would be:
[93, 60, 82, 53, 16, 42, 0, 68, 88, 11, 89, 62, 38, 14, 27, 8, 45, 25, 83, 97, 94]
[30, 5, 19, 11, 14, 6, 7, 86, 16, 53, 71, 12, 90, 32]
You can try something like this, which would randomly generate the size between 1 and 10.
import random
list1 = random.sample(range(100), random.randint(1,10))
list2 = random.sample(range(100), random.randint(1,10))
print(list1)
print(list2)
This will generate random length of the lists. Hope it helps !
You need to randomize the second Parameter as well to become lists of random size:
import random
list1 = random.sample(range(100), random.randint(1,10))
list2 = random.sample(range(100), random.randint(1,10))
print(list1)
print(list2)
how can i do this using simple code in python3
matrix = [[98, 19, 1, 46, 51, 33, 3, 33, 80, 40], [26, 88, 79, 10, 63, 76, 18, 49, 47, 44], [18, 53, 8, 96, 40, 53, 73, 8, 31, 43], [8, 40, 31, 98, 19, 39, 15, 9, 58, 32], [76, 45, 1, 5, 15, 14, 20, 88, 51, 48]
You can flatten your list via itertools.chain.from_iterable, then get the largest even and odd using %==0 for evens and %!=0 for odds:
import itertools
flat = list(itertools.chain.from_iterable(matrix))
even_max = max(i for i in flat if i%2==0)
odd_max = max(i for i in flat if i%2!=0)
>>> even_max
98
>>> odd_max
79
If you prefer to avoid itertools, you can flatten your 2d matrix by list comprehension:
flat = [v for i in matrix for v in i]
even_max = max(i for i in flat if i%2==0)
odd_max = max(i for i in flat if i%2!=0)
More options, with numpy (might be overkill for your matrix, but could be beneficial if your matrix were huge):
import numpy as np
m = np.array(matrix)
even_max = max(m[m%2==0])
odd_max = max(m[m%2!=0])
list = [1,2,,3,4,5,6,1,2,56,78,45,90,34]
range = ["0-25","25-50","50-75","75-100"]
I am coding in python. I want to sort a list of integers in range of numbers and store them in differrent lists.How can i do it?
I have specified my ranges in the the range list.
Create a dictionary with max-value of each bin as key. Iterate through your numbers and append them to the list that's the value of each bin-key:
l = [1,2,3,4,5,6,1,2,56,78,45,90,34]
# your range covers 25 a piece - and share start/endvalues.
# I presume [0-25[ ranges
def inRanges(data,maxValues):
"""Sorts elements of data into bins that have a max-value. Max-values are
given by the list maxValues which holds the exclusive upper bound of the bins."""
d = {k:[] for k in maxValues} # init all keys to empty lists
for n in data:
key = min(x for x in maxValues if x>n) # get key
d[key].append(n) # add number
return d
sortEm = inRanges(l,[25,50,75,100])
print(sortEm)
print([ x for x in sortEm.values()])
Output:
{25: [1, 2, 3, 4, 5, 6, 1, 2], 50: [25, 45, 34],
75: [56], 100: [78, 90]}
[[1, 2, 3, 4, 5, 6, 1, 2], [25, 45, 34], [56], [78, 90]]
Another stable bin approach for your special case (regular intervaled bins) would be to use a calculated key - this would get rid of the key-search in each step.
Stable search means the order of numbers in the list is the same as in the input data:
def inRegularIntervals(data, interval):
"""Sorts elements of data into bins of regular sizes.
The size of each bin is given by 'interval'."""
# init dict so keys are ordered - collection.defaultdict(list)
# would be faster - but this works for lists of a couple of
# thousand numbers if you have a quarter up to one second ...
# if random key order is ok, shorten this to d = {}
d = {k:[] for k in range(0, max(data), interval)}
for n in data:
key = n // interval # get key
key *= interval
d.setdefault(key, [])
d[key ].append(n) # add number
return d
Use on random data:
from random import choices
data = choices(range(100), k = 50)
data.append(135) # add a bigger value to see the gapped keys
binned = inRegularIntervals(data, 25)
print(binned)
Output (\n and spaces added):
{ 0: [19, 9, 1, 0, 15, 22, 4, 9, 12, 7, 12, 9, 16, 2, 7],
25: [25, 31, 37, 45, 30, 48, 44, 44, 31, 39, 27, 36],
50: [50, 50, 58, 60, 70, 69, 53, 53, 67, 59, 52, 64],
75: [86, 93, 78, 93, 99, 98, 95, 75, 88, 82, 79],
100: [],
125: [135], }
To sort the binned lists in place, use
for k in binned:
binned[k].sort()
to get:
{ 0: [0, 1, 2, 4, 7, 7, 9, 9, 9, 12, 12, 15, 16, 19, 22],
25: [25, 27, 30, 31, 31, 36, 37, 39, 44, 44, 45, 48],
50: [50, 50, 52, 53, 53, 58, 59, 60, 64, 67, 69, 70],
75: [75, 78, 79, 82, 86, 88, 93, 93, 95, 98, 99],
100: [],
125: [135]}
my code:
def originalList = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100]
def newList = orginalList.percent(0.05,0.95) //I have no idea what I'm doing here
println newList
I have an original list of numbers, they are 1 - 100 and i want to make a new list from the original list however the new list must only have data that belongs to the sub-range 5%- 95% of the original list
so the new list must be like [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18....95]. How do i do that? i know my newList code is wrong
You mean like:
originalList[ 4..94 ] // zero starting pos
Or do you need percentages?
You could do:
originalList[ (originalList.size() * 0.05 - 1)..<(originalList.size() * 0.95) ]
You could also use the metaClass:
List.metaClass.percent { double lower, double upper ->
int d = lower * delegate.size() - 1
int t = upper * delegate.size()
delegate.take( t ).drop( d )
}
originalList.percent( 0.05, 0.95 )