Related
I am working on a program that reads logfiles in, in a weird format.
I have the following code reading from a number of logfiles
(logFiles)
It works fine to itterate through them and so on. The problem is that for some reason the last element that i append to the list dataList overwrites the entire list so all fields are the same..
def fetchDataFromLogFiles(colIndex, colNames, colType, logFiles):
dataList = []
rowchange = []
row = []
for i in range(0, len(colIndex)):
row.append(0)
rowchange.append(0)
for x in range(0, len(logFiles)):
print("fetching data from: ")
print(logFiles[x])
logFile = open(logFiles[x] , "r")
for x in range(0,4):
line = logFile.readline()
if not line:
break
else:
line = line.split(",")
TimeStamp = line[1]
rest = line[2:]
#TimeDate formatting
whiteSpaces = len(TimeStamp)- len(TimeStamp.lstrip())
TimeStamp = TimeStamp[whiteSpaces:]
TimeStampFormat = "%a %b %d %H:%M:%S %Y"
DT = datetime.datetime.strptime(TimeStamp, TimeStampFormat)
#print(type(DT))
#print(DT)
row[0] = DT
rowchange[0] +=1
for i in range(0, len(rest)):
rest[i] = rest[i].replace(" ","")
if(i%2 == 0):
try:
index = colIndex.index(rest[i])
rowchange[index] += 1
if(colType[index] == "Bool"):
if(rest[i+1] == "0"):
row[index] = False
#print(type(row[index]))
else:
row[index] = True
#print(type(row[index]))
elif(colType[index] == "Short"):
row[index] = int(rest[i+1])
#print(type(row[index]))
else:
row[index] = rest[i+1]
except:
continue
print(x)
print(row[0])
dataList.append(row)
print(dataList[x][0])
logFile.close()
print("file closed")
for x in range (0,len(dataList)):
print(dataList[x][0])
return dataList
Debug print output as follows to show what i mean
0
2022-01-24 04:57:39
2022-01-24 04:57:39
1
2022-01-24 04:57:39
2022-01-24 04:57:39
2
2022-01-24 04:57:39
2022-01-24 04:57:39
3
2022-01-24 04:57:40
2022-01-24 04:57:40
file closed
2022-01-24 04:57:40
2022-01-24 04:57:40
2022-01-24 04:57:40
2022-01-24 04:57:40
A couple of lines of log file:
$1, Mon Jan 24 04:57:39 2022,$2,10,$3,42,$4, 0,$5, 1, #0, 42, #1, 130, #2, 144, #3, 49, #4, 45, #5, 66, #6, 2975, #7, -3981, #8, 4028, #9, 3900, #10, 2480, #13, 4029, #16, 2930, #17, 2190, #20, 102, #21, 2, #24, 2900, #25, 2200, #26, 6, #27, 51, #30, 2898, #31, 0, #32, 0, #33, 150, #34, 511, #35, -2, #36, 22, #37, 549, #38, -2, #39, 22, #40, 60, #41, 45, #42, 218, #43, -306, #45, 236, #46, -152, #48, 100, #49, 0, #50, 100, #51, 0, #52, 137, #53, -200, #55, 137, #56, -64, #58, 7850, #59, 1, #60, 8300, #61, 1, #62, 100, #63, -1, #64, 60, #65, 88, #66, 86, #67, 108, #68, 1, #73, 1800, #74, 2500, #76, 0, #77, 0, #78, 0, #79, 0, #80, 0, #81, 173, #82, 174, #83, 0, #84, 0, #85, -11, #86, -100, #88, 400, #89, 2729, #90, 0, #91, 2762, #93, 5, #94, 15, #95, 11, #96, 99, #97, 30, #98, 0, #100, 0, #101, 0, #102, 0,#105, 1252, #106, 0, #107, 0, #108, 0,#109, 4029, #110, 0,#111, 3900,#112, 2480,#113, 2520,#114, 2200,#115, 2900, #116, 1, #117, 0, #118, 0, #119, 156, #120, 165, #121, 0, #123, 0, #124, 0, #126, 40, #127, 60, #128, 350, #129, 450, #130, 57,#131, 1740,#132, 1682, #133, 1, #134, 0, #135, 0, #136, 0, #137, 2, #138, 0, #141, 135, #142, 0,#150, 4800, #151, 0, #152, 0,#153, 4864, #154, 1, #155, 1, #156, 0, #157, 0, #158, 1, #160, 15, #161, 40, #162, 15, #163, 40, #164, 1, #165, 1, #166, 0,#170, 1000,#172, 3000, #173, 0, #175, 0, #179, 0, #180, 599, #181, 0, #182, 9, #183, 42, #184, 0, #185, 7, #186, 49, #187, 0, #188, 4, #189, 1, #190, 1,#191, 2261,#192, 1796, #193, 1, #194, 0, #197, 0,#199, 2000,#200, 10000, #201, 10, #202, 10,#203, 3000, #204, 0, #205, 900, #206, 0,#209, 1774, #210, 300, #211, 900,#212, 1800, #213, 100,#217, 3000,#218, 4500,#219, 1500,#220, 3000, #222, 60, #223, 9,#224, 3150, #225, 0, #226, 0, #231, 30, #232, 1, #233, 0, #234, 0, #235, 100, #236, 0, #237, 0, #240, 34, #241, 29, #251, 0,
$1, Mon Jan 24 04:57:39 2022,$2,10,$3,42,$4, 0,$5, 1, #8, 4029, #30, 2897, #82, 173, #141, 132,#153, 5120, #185, 8,
$1, Mon Jan 24 04:57:39 2022,$2,10,$3,42,$4, 0,$5, 1, #30, 2898, #91, 2761, #130, 17,#131, 1701,#132, 1683, #141, 130,#153, 5376, #155, 0, #185, 9,
$1, Mon Jan 24 04:57:40 2022,$2,10,$3,42,$4, 0,$5, 1, #7, -3982, #8, 4028, #89, 2731, #141, 128,#153, 5632, #185, 8,
After Ethans comment i edited the code to not use globals, but still have the same problem
b=[2, 3, 0, 5, 0, 7, 0, 0, 0, 11, 0, 13, 0, 0, 0, 17, 0, 19, 0, 0, 0, 23, 0, 0, 0, 0, 0, 29, 0, 31, 0, 0, 0, 0, 0, 37, 0, 0, 0, 41, 0, 43, 0, 0, 0, 47, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 59, 0, 61, 0, 0, 0, 0, 0, 67, 0, 0, 0, 71, 0, 73, 0, 0, 0, 0, 0, 79, 0, 0, 0, 83, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0]
#b is a list of primes with composite numbers turned into zero!
#code to remove zeroes
for k in b:
if k==0:
del k
print(b)
I'm welcome to any suggestions that are reasonably simple since I'm a beginner and self-taught. If there's anything I'm doing horribly wrong, please do point that out as well. Thank You
I would use list comprehension to remove all occurences of 0:
b=[2, 3, 0, 5, 0, 7, 0, 0, 0, 11, 0, 13, 0, 0, 0, 17, 0, 19, 0, 0, 0, 23, 0, 0, 0, 0, 0, 29, 0, 31, 0, 0, 0, 0, 0, 37, 0, 0, 0, 41, 0, 43, 0, 0, 0, 47, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 59, 0, 61, 0, 0, 0, 0, 0, 67, 0, 0, 0, 71, 0, 73, 0, 0, 0, 0, 0, 79, 0, 0, 0, 83, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0]
#b is a list of primes with composite numbers turned into zero!
# Iterates through all the values and returns them unless they are 0
b = [x for x in b if not x == 0]
print(b)
I have multiple npz files which i want to merge into one npz.file with the format similar to "mnist.npz"
the format of mnist.npz is:
((array([[[0, 0, 0, ..., 0, 0, 0],
[0, 0, 0, ..., 0, 0, 0],
[0, 0, 0, ..., 0, 0, 0],
...,
[0, 0, 0, ..., 0, 0, 0],
[0, 0, 0, ..., 0, 0, 0],
[0, 0, 0, ..., 0, 0, 0]],
[0, 0, 0, ..., 0, 0, 0]]], dtype=uint8),
array([5, 0, 4, ..., 5, 6, 8], dtype=uint8))
Here two arrays are merged into one big npz file.
My two npz arrays are:
x_array:
[[[252, 251, 253],
[151, 150, 152],
[ 28, 25, 27],
...,
[ 30, 25, 27],
[ 30, 25, 27],
[ 32, 27, 29]],
[ 23, 18, 20]],
[[ 50, 92, 163],
[ 55, 90, 163],
[ 75, 105, 176],
...,
[148, 197, 242],
[109, 157, 208],
[109, 165, 222]],
[[ 87, 104, 155],
[ 82, 112, 168],
...,
[ 29, 52, 105],
[ 30, 55, 111],
[ 36, 55, 106]]]
y_array:
[1, 1, 1, 1, 1, 1]
When i tried to merge my files, the output i got is:
(array([[[252, 251, 253],
[151, 150, 152],
[ 28, 25, 27],
...,
[ 30, 25, 27],
[ 30, 25, 27],
[ 32, 27, 29]],
[ 23, 18, 20]]], dtype=uint8), array([[[ 50, 92, 163],
[ 55, 90, 163],
[ 75, 105, 176],
...,
[148, 197, 242],
[109, 157, 208],
[109, 165, 222]],
[ 87, 104, 155],
[ 82, 112, 168],
...,
[ 29, 52, 105],
[ 30, 55, 111],
[ 36, 55, 106]]], dtype=uint8),1, 1, 1, 1, 1, 1)
So in the last line, my array is formated as
1, 1, 1, 1, 1, 1
instead of something like:
array([1, 1, 1, 1, 1, 1], dtype=uint8)
My code for merging two npz files is:
data = load('x_array.npz',allow_pickle=True)
lst = data.files
for item in lst:
x_train = data[item]
#print((x_item,x_train))
data1 = load('y_array.npz',allow_pickle=True)
lst1 = data1.files
for item in lst1:
y_train = data1[item]
out1 = (*x_train,*y_train)
np.savez('out1.npz',out1)
print(out1)
Can anyone please suggest how i can convert my second array of (1, 1, 1, 1, 1, 1) to array([1, 1, 1, 1, 1, 1], dtype=uint8)? Any suggestions are helpful
After going through my code i found out that by changing the line
out1 = (*x_train,*y_train)
to
out1 = (*x_train,y_train)
i have list with numbers and i want to slice all the elements between numbers 192 that exist on the list and pass them to a list
my list
[192, 0, 1, 0, 1, 192, 12, 0, 5, 0, 1, 0, 1, 66, 218, 0, 10, 5, 115, 116, 97, 116, 115, 1, 108, 192, 20, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 155, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 156, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 154, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 157]
i want someting like this
[192, 0, 1, 0, 1 ]
[192, 12, 0, 5, 0, 1, 0, 1, 66, 218, 0, 10, 5, 115, 116, 97, 116, 115, 1, 108]
[192, 20, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 155]
until the end of the list.
Here's one possible way to do it:
# input list
lst = [192, 0, 1, 0, 1, 192, 12, 0, 5, 0, 1, 0, 1, 66, 218, 0, 10, 5, 115, 116, 97, 116, 115, 1, 108, 192, 20, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 155, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 156, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 154, 192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 157]
# list of indexes where 192 is found,
# plus one extra index for the final slice
indexes = [i for i, n in enumerate(lst) if n == 192] + [len(lst)]
# create the slices between consecutive indexes
[lst[indexes[i]:indexes[i+1]] for i in range(len(indexes) - 1)]
The result will be:
[[192, 0, 1, 0, 1],
[192, 12, 0, 5, 0, 1, 0, 1, 66, 218, 0, 10, 5, 115, 116, 97, 116, 115, 1, 108],
[192, 20],
[192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 155],
[192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 156],
[192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 154],
[192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 157]]
You can build a generator with itertools.groupby that uses 192's equality method as a key function, pair the output of the generator with zip and then use itertools.chain.from_iterable to join the pairs (the example below assumes your list is stored in variable l):
from itertools import groupby, chain
i = (list(g) for _, g in groupby(l, key=(192).__eq__))
[list(chain.from_iterable(p)) for p in zip(i, i)]
This returns:
[[192, 0, 1, 0, 1],
[192, 12, 0, 5, 0, 1, 0, 1, 66, 218, 0, 10, 5, 115, 116, 97, 116, 115, 1, 108],
[192, 20],
[192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 155],
[192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 156],
[192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 154],
[192, 53, 0, 1, 0, 1, 0, 0, 0, 162, 0, 4, 74, 125, 133, 157]]
I am using this library to read png images, but in general
https://www.npmjs.com/package/pngjs2
if I have a Uint8ClampedArray, a width and height of the array, how can I convert that and save it as a png image in node.js?
Thanks
You can use the same library to create a png image if the dimensions are known and the data is in the form of a Uint8ClampedArray. Example:
var fs = require('fs'),
PNG = require('pngjs2').PNG;
var img_width = 16;
var img_height = 16;
var img_data = Uint8ClampedArray.from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 133, 110, 6, 97, 137, 82, 249, 97, 142, 79, 255, 93, 142, 74, 255, 90, 140, 71, 255, 90, 142, 70, 255, 79, 129, 60, 250, 115, 134, 92, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 152, 125, 15, 111, 151, 96, 255, 223, 255, 209, 255, 174, 253, 148, 255, 158, 249, 126, 255, 141, 249, 103, 255, 71, 145, 43, 255, 68, 143, 42, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 158, 131, 20, 111, 153, 96, 255, 216, 255, 201, 255, 172, 247, 145, 255, 156, 244, 124, 255, 139, 242, 102, 255, 72, 145, 44, 255, 75, 144, 47, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 158, 131, 25, 110, 154, 94, 255, 196, 252, 178, 255, 157, 242, 125, 255, 144, 239, 110, 255, 129, 237, 91, 255, 70, 145, 42, 255, 70, 142, 43, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 153, 128, 30, 107, 155, 90, 255, 177, 245, 151, 255, 134, 233, 100, 255, 125, 230, 87, 255, 114, 229, 73, 255, 69, 146, 41, 255, 66, 140, 40, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 154, 120, 55, 103, 155, 83, 255, 154, 236, 125, 255, 111, 223, 71, 255, 109, 222, 69, 255, 109, 225, 69, 255, 69, 146, 40, 255, 63, 133, 41, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 142, 107, 82, 100, 154, 79, 255, 145, 229, 114, 255, 103, 218, 62, 255, 105, 218, 65, 255, 106, 220, 66, 255, 69, 145, 39, 255, 67, 125, 49, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 119, 56, 249, 126, 178, 106, 255, 122, 174, 104, 255, 128, 194, 105, 255, 140, 226, 109, 255, 105, 215, 65, 255, 103, 214, 63, 255, 104, 215, 63, 255, 84, 167, 53, 255, 78, 139, 54, 255, 78, 142, 54, 255, 71, 127, 50, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 91, 8, 240, 63, 157, 29, 255, 134, 222, 103, 255, 153, 229, 124, 255, 166, 233, 140, 255, 110, 213, 73, 255, 100, 210, 61, 255, 100, 210, 61, 255, 125, 221, 91, 255, 124, 221, 89, 255, 78, 179, 43, 255, 54, 122, 29, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 78, 7, 42, 39, 106, 14, 253, 60, 156, 26, 255, 120, 210, 88, 255, 127, 217, 96, 255, 119, 214, 85, 255, 96, 207, 56, 255, 98, 209, 59, 255, 95, 204, 56, 255, 70, 166, 37, 255, 57, 131, 30, 253, 48, 108, 24, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 87, 9, 42, 42, 107, 18, 249, 63, 160, 28, 255, 107, 201, 73, 255, 121, 212, 88, 255, 108, 210, 72, 255, 87, 194, 50, 255, 62, 153, 30, 255, 50, 118, 25, 249, 43, 103, 23, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 93, 13, 38, 46, 109, 22, 241, 68, 166, 32, 255, 99, 197, 63, 255, 89, 185, 55, 255, 54, 141, 24, 255, 44, 108, 20, 241, 36, 93, 19, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 97, 12, 26, 47, 110, 25, 230, 68, 162, 37, 255, 46, 127, 17, 255, 39, 98, 16, 230, 33, 89, 13, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 93, 21, 16, 50, 112, 26, 223, 41, 101, 19, 225, 20, 72, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
var img_png = new PNG({width: img_width, height: img_height})
img_png.data = Buffer.from(img_data);
img_png.pack().pipe(fs.createWriteStream('tick.png'))