from socket import *
import packets
image = "testfile.bmp"
# open image
bufferSize = 2048
myfile = open(image, 'rb')
sequenceNumber = 0
totalBytes = 0
serverName = "127.0.0.1"
serverSendPort = 12000
serverSendPort = int(serverSendPort)
serverListenPort = 12001
clientSocket = socket(AF_INET, SOCK_DGRAM)
serverSocket = socket(AF_INET, SOCK_DGRAM)
while (1):
if (sequenceNumber == 0):
data = packets.mkepckt(myfile.read(bufferSize), 0,
packets.calculateChecksum(myfile.read(bufferSize), bufferSize))
clientSocket.sendto(bytearray(data),(serverName, serverSendPort))
sequenceNumber = 1
ACK, = serverSocket.recvfrom(bufferSize)
ACKchecksum = packets.calculateChecksum(ACK[1], bufferSize)
while (ACK[0] is not 0 or ACKchecksum is not ACK[2]):
clientSocket.sendto(data, (serverName, serverSendPort))
ACK, = serverSocket.recvfrom(bufferSize)
ACKchecksum = packets.calculateChecksum(ACK[1], bufferSize)
elif (sequenceNumber == 1):
data = packets.mkepckt(myfile.read(bufferSize), 1,
packets.calculateChecksum(myfile.read(bufferSize), bufferSize))
clientSocket.sendto(bytearray(data),(serverName, serverSendPort))
sequenceNumber = 0
ACK, = serverSocket.recvfrom(bufferSize)
ACKchecksum = packets.checksum(bufferSize, ACK[1])
while (ACK[0] is not 1 or ACKchecksum is not ACK[2]):
clientSocket.sendto(data, (serverName, serverSendPort))
ACK, = serverSocket.recvfrom(bufferSize)
ACKchecksum = packets.calculateChecksum(ACK[1], bufferSize)
myfile.close()
clientSocket.close()
serverSocket.close()
Not sure I needed to put my whole code, but better safe than sorry.
So for some reason, despite the fact that I have an int in the sendto. I have also tried all sorts of typecasting, from typecasting each individual element to typecasting the whole thing as an int, and I still get this error.
Thanks to help from #RemyLebeau, I was able to solve this problem by importing pickle and using that to dump my tuple into a variable, then sending that variable as the data. Not sure why the byte data gave off an error about an int, but there you go.
Related
I would like to create a .tar file in an S3 bucket from Python code running in an AWS Lambda function. Lambda functions are very memory- and disk- constrained. I want to create a .tar file that contains multiple files that are too large to fit in the Lambda function's memory or disk space.
Using "S3 multipart upload," it is possible to upload a large file by uploading chunks of 5MB or more in size. I have this figured out and working. What I need to figure out is how to manage a buffer of bytes in memory that won't grow past the limits of the Lambda function's runtime environment.
I think the solution is to create an io.BytesIO() object and manage both a read pointer and a write pointer. I can then write into the buffer (from files that I want to add to the .tar file) and every time the buffer exceeds some limit (like 5MB) I can read off a chunk of data and send another file part to S3.
What I haven't quite wrapped my head around is how to truncate the part of the buffer that has been read and is no longer needed in memory. I need to trim the head of the buffer, not the tail, so the truncate() function of BytesIO won't work for me.
Is the 'correct' solution to create a new BytesIO buffer, populating it with the contents of the existing buffer from the read pointer to the end of the buffer, when I truncate? Is there a better way to truncate the head of the BytesIO buffer? Is there a better solution than using BytesIO?
For the random Google-r who stumbles onto this question six years in the future and thinks, "man, that describes my problem exactly!", here's what I came up with:
import io
import struct
from tarfile import BLOCKSIZE
#This class was designed to write a .tar file to S3 using multipart upload
#in a memory- and disk constrained environment, such as AWS Lambda Functions.
#
#Much of this code is copied or adapted from the Python source code tarfile.py
#file at https://github.com/python/cpython/blob/3.10/Lib/tarfile.py
#
#No warranties expressed or implied. Your mileage may vary. Lather, rinse, repeat
class StreamingTarFileWriter:
#Various constants from tarfile.py that we need
GNU_FORMAT = 1
NUL = b"\0"
BLOCKSIZE = 512
RECORDSIZE = BLOCKSIZE * 20
class MemoryByteStream:
def __init__(self, bufferFullCallback = None, bufferFullByteCount = 0):
self.buf = io.BytesIO()
self.readPointer = 0
self.writePointer = 0
self.bufferFullCallback = bufferFullCallback
self.bufferFullByteCount = bufferFullByteCount
def write(self, buf: bytes):
self.buf.seek(self.writePointer)
self.writePointer += self.buf.write(buf)
bytesAvailableToRead = self.writePointer - self.readPointer
if self.bufferFullByteCount > 0 and bytesAvailableToRead > self.bufferFullByteCount:
if self.bufferFullCallback:
self.bufferFullCallback(self, bytesAvailableToRead)
def read(self, byteCount = None):
self.buf.seek(self.readPointer)
if byteCount:
chunk = self.buf.read(byteCount)
else:
chunk = self.buf.read()
self.readPointer += len(chunk)
self._truncate()
return chunk
def size(self):
return self.writePointer - self.readPointer
def _truncate(self):
self.buf.seek(self.readPointer)
self.buf = io.BytesIO(self.buf.read())
self.readPointer = 0
self.writePointer = self.buf.seek(0, 2)
def stn(self, s, length, encoding, errors):
#Convert a string to a null-terminated bytes object.
s = s.encode(encoding, errors)
return s[:length] + (length - len(s)) * self.NUL
def itn(self, n, digits=8, format=GNU_FORMAT):
#Convert a python number to a number field.
# POSIX 1003.1-1988 requires numbers to be encoded as a string of
# octal digits followed by a null-byte, this allows values up to
# (8**(digits-1))-1. GNU tar allows storing numbers greater than
# that if necessary. A leading 0o200 or 0o377 byte indicate this
# particular encoding, the following digits-1 bytes are a big-endian
# base-256 representation. This allows values up to (256**(digits-1))-1.
# A 0o200 byte indicates a positive number, a 0o377 byte a negative
# number.
original_n = n
n = int(n)
if 0 <= n < 8 ** (digits - 1):
s = bytes("%0*o" % (digits - 1, n), "ascii") + self.NUL
elif format == self.GNU_FORMAT and -256 ** (digits - 1) <= n < 256 ** (digits - 1):
if n >= 0:
s = bytearray([0o200])
else:
s = bytearray([0o377])
n = 256 ** digits + n
for i in range(digits - 1):
s.insert(1, n & 0o377)
n >>= 8
else:
raise ValueError("overflow in number field")
return s
def calc_chksums(self, buf):
"""Calculate the checksum for a member's header by summing up all
characters except for the chksum field which is treated as if
it was filled with spaces. According to the GNU tar sources,
some tars (Sun and NeXT) calculate chksum with signed char,
which will be different if there are chars in the buffer with
the high bit set. So we calculate two checksums, unsigned and
signed.
"""
unsigned_chksum = 256 + sum(struct.unpack_from("148B8x356B", buf))
signed_chksum = 256 + sum(struct.unpack_from("148b8x356b", buf))
return unsigned_chksum, signed_chksum
def __init__(self, bufferFullCallback = None, bufferFullByteCount = 0):
self.buf = self.MemoryByteStream(bufferFullCallback, bufferFullByteCount)
self.expectedFileSize = 0
self.fileBytesWritten = 0
self.offset = 0
pass
def addFileRecord(self, filename, filesize):
REGTYPE = b"0" # regular file
encoding = "utf-8"
LENGTH_NAME = 100
GNU_MAGIC = b"ustar \0" # magic gnu tar string
errors="surrogateescape"
#Copied from TarInfo.tobuf()
tarinfo = {
"name": filename,
"mode": 0o644,
"uid": 0,
"gid": 0,
"size": filesize,
"mtime": 0,
"chksum": 0,
"type": REGTYPE,
"linkname": "",
"uname": "",
"gname": "",
"devmajor": 0,
"devminor": 0,
"magic": GNU_MAGIC
}
buf = b""
if len(tarinfo["name"].encode(encoding, errors)) > LENGTH_NAME:
raise Exception("Filename is too long for tar file header.")
devmajor = self.stn("", 8, encoding, errors)
devminor = self.stn("", 8, encoding, errors)
parts = [
self.stn(tarinfo.get("name", ""), 100, encoding, errors),
self.itn(tarinfo.get("mode", 0) & 0o7777, 8, self.GNU_FORMAT),
self.itn(tarinfo.get("uid", 0), 8, self.GNU_FORMAT),
self.itn(tarinfo.get("gid", 0), 8, self.GNU_FORMAT),
self.itn(tarinfo.get("size", 0), 12, self.GNU_FORMAT),
self.itn(tarinfo.get("mtime", 0), 12, self.GNU_FORMAT),
b" ", # checksum field
tarinfo.get("type", REGTYPE),
self.stn(tarinfo.get("linkname", ""), 100, encoding, errors),
tarinfo.get("magic", GNU_MAGIC),
self.stn(tarinfo.get("uname", ""), 32, encoding, errors),
self.stn(tarinfo.get("gname", ""), 32, encoding, errors),
devmajor,
devminor,
self.stn(tarinfo.get("prefix", ""), 155, encoding, errors)
]
buf = struct.pack("%ds" % BLOCKSIZE, b"".join(parts))
chksum = self.calc_chksums(buf[-BLOCKSIZE:])[0]
buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:]
self.buf.write(buf)
self.expectedFileSize = filesize
self.fileBytesWritten = 0
self.offset += len(buf)
def addFileData(self, buf):
self.buf.write(buf)
self.fileBytesWritten += len(buf)
self.offset += len(buf)
pass
def completeFileRecord(self):
if self.fileBytesWritten != self.expectedFileSize:
raise Exception(f"Expected {self.expectedFileSize:,} bytes but {self.fileBytesWritten:,} were written.")
#write the end-of-file marker
blocks, remainder = divmod(self.fileBytesWritten, BLOCKSIZE)
if remainder > 0:
self.buf.write(self.NUL * (BLOCKSIZE - remainder))
self.offset += BLOCKSIZE - remainder
def completeTarFile(self):
self.buf.write(self.NUL * (BLOCKSIZE * 2))
self.offset += (BLOCKSIZE * 2)
blocks, remainder = divmod(self.offset, self.RECORDSIZE)
if remainder > 0:
self.buf.write(self.NUL * (self.RECORDSIZE - remainder))
An example use of the class is:
OUTPUT_CHUNK_SIZE = 1024 * 1024 * 5
f_out = open("test.tar", "wb")
def get_file_block(blockNum):
block = f"block_{blockNum:010,}"
block += "0123456789abcdef" * 31
return bytes(block, 'ascii')
def buffer_full_callback(x: StreamingTarFileWriter.MemoryByteStream, bytesAvailable: int):
while x.size() > OUTPUT_CHUNK_SIZE:
buf = x.read(OUTPUT_CHUNK_SIZE)
#This is where you would write the chunk to S3
f_out.write(buf)
x = StreamingTarFileWriter(buffer_full_callback, OUTPUT_CHUNK_SIZE)
import random
numFiles = random.randint(3,8)
print(f"Creating {numFiles:,} files.")
for fileIdx in range(numFiles):
minSize = 1025 #1kB plus 1 byte
maxSize = 10 * 1024 * 1024 * 1024 + 5 #10GB plus 5 bytes
numBytes = random.randint(minSize, maxSize)
print(f"Creating file {str(fileIdx)} with {numBytes:,} bytes.")
blocks,remainder = divmod(numBytes, 512)
x.addFileRecord(f"File{str(fileIdx)}", numBytes)
for block in range(blocks):
x.addFileData(get_file_block(block))
x.addFileData(bytes(("X" * remainder), 'ascii'))
x.completeFileRecord()
I get an Connection reset by peer error message while running this python3 script:
import struct, socket
HOST = "127.0.0.1"
PORT = 64012
# read bytes from socket and return it as ascii representation
def read_line_from_socket(s):
buf= ""
while True:
byteData = s.recv(1)
if byteData == b"\n":
break
buf += byteData.decode("ascii")
return buf
def read_four_longs_for_addition(s):
sum = 0
counter = 1
while counter <= 4:
sum += struct.unpack("I", s.recv(4))[0]
counter += 1 # python does not support ++, += 1 has to be used
print(sum) # sum overflows int (number > 32 bit)
return sum
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST,PORT))
print(read_line_from_socket(s))
print(read_line_from_socket(s))
s.send(struct.pack("l", read_four_longs_for_addition(s)))
# print(s.recv(32)) # receives "You have succes..", NOT resulting in an Connection reset by peer which indicates success
print(read_line_from_socket(s)) # results in "Connection reset by peer"
s.close()
The code on the server which is communicating with my client socket looks like this:
#define BANNER \
"Welcome to " LEVELNAME ", brought to you by https://exploit.education"
int main(int argc, char **argv) {
int i;
unsigned long quad[sizeof(long)], result, wanted;
setvbuf(stdout, NULL, _IONBF, 0);
setvbuf(stderr, NULL, _IONBF, 0);
printf("%s\nFor this level, sizeof(long) == %d, keep that in mind :)\n",
BANNER, (int)sizeof(long));
if (getrandom((void *)&quad, sizeof(quad), 0) != sizeof(quad)) {
errx(1, "unable to getrandom(%d bytes)", sizeof(quad));
}
result = 0;
for (i = 0; i < sizeof(long); i++) {
result += quad[i];
if (write(1, (void *)&quad[i], sizeof(long)) != sizeof(long)) {
errx(1, "Why have you foresaken me, write()");
}
}
if (read(0, (void *)&wanted, sizeof(long)) != sizeof(long)) {
errx(1, "Unable to read\n");
}
if (result == wanted) {
printf("You have successfully passed this level, well done!\n");
} else {
printf("Whoops, better luck next time. Receieved %lu, wanted %lu\n", wanted,
result);
}
return 0;
}
Expected behavior:
After successfully sending the correct sum of the received long values my script reads the last line "You have successfully passed this level, well done!\n" until the "\n" new line, prints it out to the screen and finally closing the socket connection.
Actual output:
Welcome to phoenix/net-two, brought to you by https://exploit.education
For this level, sizeof(long) == 4, keep that in mind :)
9057758327
Traceback (most recent call last):
File "net-2.py", line 31, in <module>
print(read_line_from_socket(s))
File "net-2.py", line 10, in read_line_from_socket
byteData = s.recv(1)
ConnectionResetError: [Errno 104] Connection reset by peer
Looking at the process with strace python3 nameOfPythonScript.py, I can't see any problem and everything should be working fine:
recvfrom(3, "n", 1, 0, NULL, NULL) = 1
recvfrom(3, "e", 1, 0, NULL, NULL) = 1
recvfrom(3, "!", 1, 0, NULL, NULL) = 1
recvfrom(3, "\n", 1, 0, NULL, NULL) = 1
write(1, "You have successfully passed thi"..., 52You have successfully passed this level, well done!
) = 52
close(3) = 0
rt_sigaction(SIGINT, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7ffff7bcd0c0}, {sa_handler=0x5555557d8980, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7ffff7bcd0c0}, 8) = 0
munmap(0x7ffff6c0e000, 262144) = 0
exit_group(0) = ?
+++ exited with 0 +++
My question:
Why does the Connection reset occur? Does it have to do with the return 0; in the server program code? Note that if I recv a fixed number of bytes instead of trying to read the complete last line the error does not occur (see comments in my script's code)
I need to define n ports for each server so that i can know if any server is connected to more than 1 client than at which port the connection is taking place.
import queue
import socket
import random
import threading
import pandas as pd
UDP_IP = "127.0.0.1"
num_server = 3
num_client = 7
server_thread_list = []
client_thread_list = []
sid = []
sip = []
dip = []
connect = []
l_info= []
s_port = []
topology_buffer = queue.Queue()
def create_csv():
source_mac = sid
source_ip = sip
dest_mac = connect
destination_ip = dip
link_info = l_info
source_portno = s_port
df = pd.DataFrame()
df["Source_MAC"] = source_mac
df["Source_IP"] = source_ip
df["Source_Port_No"] = source_portno
df["Dest_MAC"] = dest_mac
df["Dest_IP"] = destination_ip
df["Link_Info"] = link_info
print(df)
df.to_csv("network.csv", index = False)
def server(id):
UDP_PORT = 5000 + id
soc = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
soc.bind((UDP_IP,UDP_PORT))
while True:
data, addr = soc.recvfrom(1024)
data = data.decode()
data1 = "S" + str(id)
print("Server" + str(id) + "recieved message from" + str(data))
topology_buffer.put_nowait(str(data1 + "<-->" + data))
for i in range(len(data1)-1):
sid.append(data1)
ser_ip = '192.168.1.' + str(random.randint(0, 255))
sip.append(ser_ip) # generate ip address
s_port.append(UDP_PORT)
conn = data
connect.append(conn)
dest_ip = '192.168.1.' + str(random.randint(0, 255))
dip.append(dest_ip)
def client(id):
if id<= len(server_thread_list):
UDP_Port = 5000 + id
else:
UDP_Port = 5000 + random.randint(0, len(server_thread_list))
soc = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
message = ("C" + str(id))
soc.sendto(message.encode(),(UDP_IP, UDP_Port))
soc.close()
create_csv()
def topology():
while True:
link = topology_buffer.get()
print(link)
for i in range (num_server):
t = threading.Thread(target = server, args=(i,))
server_thread_list.append(t)
t.start()
for i in range (num_client):
t = threading.Thread(target = client, args=(i,))
client_thread_list.append(t)
t.start()
topology()
I want the result as, the port number to be printed along with the connection.
eg: Server S1a received message from c1 at port number 5002.
Server S1b received message from c2 at port number 5003.
Server S2b received message from c3 at port number 5004.
I'm trying to upgrade an old Python 2.x class to work with 3.x (written by someone else, and not maintained). Here is a fragment.
def getByte(self):
if (self.available < 1):
self.request(4096);
self.available = self.available - 1
result = ord(self.bytes[0]);
self.bytes = self.bytes[1:]
return result
def request(self, size):
sock = socket(AF_INET, SOCK_STREAM)
sock.connect((self.host, self.port))
contentSize = len(self.user)+len(self.password)+6;
contentString = "!BHB"+str(len(self.user))+"sB"+str(len(self.password))+"sL"
data = pack(contentString, 0, contentSize, len(self.user), self.user, len(self.password), self.password, size)
sock.sendall(data)
print("Sent!")
data = sock.recv(6)
fields = unpack("!BBL", str(data));
if (fields[0] != 0):
raise(ServiceDeniedError(fields[0],fields[1]))
self.bytes = sock.recv(fields[2]);
self.available = len(self.bytes);
print("Received: ")
print(self.available)
sock.close()
Imported and called with print (rand.getByte()).
Tracebacks:
print (rand.getByte())
self.request(4096);
data = pack(contentString, 0, contentSize, len(self.user), self.user, len(self.password), self.password, size)
struct.error: argument for 's' must be a bytes object
I figured it out. Two arguments in the pack() function had to be converted to bytes.
data = pack(contentString, 0, contentSize, len(self.user), bytes(self.user, 'utf-8'), len(self.password), bytes(self.password, 'utf-8'), size)
trying to search within a .thumbdata3 file for thumbnail images. This was someone else's sample code, but I am getting an error
"str does not support the buffer interface using .find"
"""extract files from Android thumbdata3 file"""
f=open('thumbdata3.dat','rb')
tdata = f.read()
f.close()
ss = '\xff\xd8'
se = '\xff\xd9'
count = 0
start = 0
while True:
x1 = tdata.find(ss,start)
if x1 < 0:
break
x2 = tdata.find(se,x1)
jpg = tdata[x1:x2+1]
count += 1
fname = 'extracted%d03.jpg' % (count)
fw = open(fname,'wb')
fw.write(jpg)
fw.close()
start = x2+2
ok, turned out to be very simple.
just add b in front of the data I am trying to match
so
ss = '\xff\xd8'
se = '\xff\xd9'
becomes
ss = b'\xff\xd8'
se = b'\xff\xd9'
It's all right.
With Python 3.x like python-3.6.2
Rename .thumbdata3-1763508120 file to thumbdata3.dat
Rename .thumbdata3--1967290299 file to thumbdata4.dat
enter code here
"""extract files from Android thumbdata3 file"""
f=open('thumbdata3.dat','rb')
tdata = f.read()
f.close()
ss = b'\xff\xd8'
se = b'\xff\xd9'
count = 0
start = 0
while True:
x1 = tdata.find(ss,start)
if x1 < 0:
break
x2 = tdata.find(se,x1)
jpg = tdata[x1:x2+1]
count += 1
fname = 'extracted%d03.jpg' % (count)
fw = open(fname,'wb')
fw.write(jpg)
fw.close()
start = x2+2
enter code here
"""extract files from Android thumbdata4 file"""
f=open('thumbdata4.dat','rb')
tdata = f.read()
f.close()
ss = b'\xff\xd8'
se = b'\xff\xd9'
count = 0
start = 0
while True:
x1 = tdata.find(ss,start)
if x1 < 0:
break
x2 = tdata.find(se,x1)
jpg = tdata[x1:x2+1]
count += 1
fname = 'extracted%d04.jpg' % (count)
fw = open(fname,'wb')
fw.write(jpg)
fw.close()
start = x2+2