How do I dump the struct into the byte array without reflection? - struct

I already found encoding/binary package to deal with it, but it depended on reflect package so it didn't work with uncapitalized(that is, unexported) struct fields. However I spent a week to find that problem out, I still have a question: if struct fields should not be exported, how do I dump them easily into binary data?
EDIT: Here's the example. If you capitalize the name of fields of Data struct, that works properly. But Data struct was intended to be an abstract type, so I don't want to export these fields.
package main
import (
"fmt"
"encoding/binary"
"bytes"
)
type Data struct {
id int32
name [16]byte
}
func main() {
d := Data{Id: 1}
copy(d.Name[:], []byte("tree"))
buffer := new(bytes.Buffer)
binary.Write(buffer, binary.LittleEndian, d)
// d was written properly
fmt.Println(buffer.Bytes())
// try to read...
buffer = bytes.NewBuffer(buffer.Bytes())
var e = new(Data)
err := binary.Read(buffer, binary.LittleEndian, e)
fmt.Println(e, err)
}

Your best option would probably be to use the gob package and let your struct implement the GobDecoder and GobEncoder interfaces in order to serialize and deserialize private fields.
This would be safe, platform independent, and efficient. And you have to add those GobEncode and GobDecode functions only on structs with unexported fields, which means you don't clutter the rest of your code.
func (d *Data) GobEncode() ([]byte, error) {
w := new(bytes.Buffer)
encoder := gob.NewEncoder(w)
err := encoder.Encode(d.id)
if err!=nil {
return nil, err
}
err = encoder.Encode(d.name)
if err!=nil {
return nil, err
}
return w.Bytes(), nil
}
func (d *Data) GobDecode(buf []byte) error {
r := bytes.NewBuffer(buf)
decoder := gob.NewDecoder(r)
err := decoder.Decode(&d.id)
if err!=nil {
return err
}
return decoder.Decode(&d.name)
}
func main() {
d := Data{id: 7}
copy(d.name[:], []byte("tree"))
buffer := new(bytes.Buffer)
// writing
enc := gob.NewEncoder(buffer)
err := enc.Encode(d)
if err != nil {
log.Fatal("encode error:", err)
}
// reading
buffer = bytes.NewBuffer(buffer.Bytes())
e := new(Data)
dec := gob.NewDecoder(buffer)
err = dec.Decode(e)
fmt.Println(e, err)
}

Related

Persistent Reader() object

In Go, I am trying to create a function that reads and processes the next line of input:
// Read a string of hex from stdin and parse to an array of bytes
func ReadHex() []byte {
r := bufio.NewReader(os.Stdin)
t, _ := r.ReadString('\n')
data, _ := hex.DecodeString(strings.TrimSpace(t))
return data
}
Unfortunately, this only works the first time it is called. It captures the first line but is unable to capture subsequent lines piped via standard input.
I suspect, if the same persistent bufio.Reader() object was used on each subsequent call, it would work but I haven't been able to achieve this without passing it manually on each function call.
Yes, try this:
package main
import (
"bufio"
"encoding/hex"
"fmt"
"log"
"os"
"strings"
)
func ReadFunc() func() []byte {
r := bufio.NewReader(os.Stdin)
return func() []byte {
t, err := r.ReadString('\n')
if err != nil {
log.Fatal(err)
}
data, err := hex.DecodeString(strings.TrimSpace(t))
if err != nil {
log.Fatal(err)
}
return data
}
}
func main() {
r, w, err := os.Pipe()
if err != nil {
log.Fatal(err)
}
os.Stdin = r
w.Write([]byte(`ffff
cafebabe
ff
`))
w.Close()
ReadHex := ReadFunc()
fmt.Println(ReadHex())
fmt.Println(ReadHex())
fmt.Println(ReadHex())
}
Output:
[255 255]
[202 254 186 190]
[255]
Using a struct, try this:
package main
import (
"bufio"
"encoding/hex"
"fmt"
"io"
"log"
"os"
"strings"
)
// InputReader struct
type InputReader struct {
bufio.Reader
}
// New creates an InputReader
func New(rd io.Reader) *InputReader {
return &InputReader{Reader: *bufio.NewReader(rd)}
}
// ReadHex returns a string of hex from stdin and parse to an array of bytes
func (r *InputReader) ReadHex() []byte {
t, err := r.ReadString('\n')
if err != nil {
log.Fatal(err)
}
data, err := hex.DecodeString(strings.TrimSpace(t))
if err != nil {
log.Fatal(err)
}
return data
}
func main() {
r, w, err := os.Pipe()
if err != nil {
log.Fatal(err)
}
os.Stdin = r
w.Write([]byte(`ffff
cafebabe
ff
`))
w.Close()
rdr := New(os.Stdin)
fmt.Println(rdr.ReadHex())
fmt.Println(rdr.ReadHex())
fmt.Println(rdr.ReadHex())
}

strings.Builder memory usage

Using this file (data file):
package main
import (
"io/ioutil"
"time"
)
func main() {
ioutil.ReadFile("100mb.file")
time.Sleep(time.Duration(time.Minute))
}
Showed memory usage for me of 107 MB. With this similar file:
package main
import (
"bytes"
"os"
"time"
)
func read(path_s string) (bytes.Buffer, error) {
buf_o := bytes.Buffer{}
open_o, e := os.Open(path_s)
if e != nil {
return buf_o, e
}
buf_o.ReadFrom(open_o)
open_o.Close()
return buf_o, nil
}
func main() {
read("100mb.file")
time.Sleep(time.Duration(time.Minute))
}
Memory usage went to 273 MB. Finally this similar file:
package main
import (
"io"
"os"
"strings"
"time"
)
func read(path_s string) (strings.Builder, error) {
str_o := strings.Builder{}
open_o, e := os.Open(path_s)
if e != nil {
return str_o, e
}
io.Copy(&str_o, open_o)
open_o.Close()
return str_o, nil
}
func main() {
read("100mb.file")
time.Sleep(time.Duration(time.Minute))
}
Memory usage went to 432 MB. I tried to be careful and close files where
possible. Why is the memory usage so high for the second example, and especially
the final example? Can I change something so that they are closer to the first
example?
ioutil.ReadFile("100mb.file") gets the size of the file, allocates a []byte that size and slurps the bytes up into that slice.
buf_o.ReadFrom(open_o) allocates an initial []byte of some size and reads into that slice. If there's more data in the reader than space in the slice, then the function allocates a larger slice, copies existing data to that slice and reads more. This repeats until EOF.
The function ioutil.ReadFile uses bytes Buffer.ReadFrom internally. Take a look at the ioutil.ReadFile implementation to see how to improve direct use of the bytes.Buffer. A synopsis of the logic is this:
var buf bytes.Buffer
// Open file
f, err := os.Open(path)
if err != nil {
return &buf, err
}
defer f.Close()
// Get size.
fi, err := f.Stat()
if err != nil {
return &buf, err
}
// Grow to size of file plus extra slop to ensure no realloc.
buf.Grow(int(fi.Size()) + bytes.MinRead)
_, err := buf.ReadFrom(f)
return &buf, err
The strings.Builder example reallocates the internal buffer several times as in the bytes.Buffer example. In addition, io.Copy allocates a buffer. You can improve the strings.Builder example by growing the builder to the size of the file before reading.
Here's the code for strings.Builder:
var buf strings.Builder
// Open file
f, err := os.Open(path)
if err != nil {
return &buf, err
}
defer f.Close()
// Get size.
fi, err := f.Stat()
if err != nil {
return &buf, err
}
buf.Grow(int(fi.Size()))
_, err = io.Copy(&buf, f)
return &buf, err
io.Copy or some other code using an extra buffer is required because strings.Builder does not have a ReadFrom method. The strings.Builder type does not have a ReadFrom method because that method can leak a reference to the backing array of the internal slice of bytes.
Using suggestion from Muffin Top, I took my second example and added this
directly before the call to ReadFrom:
stat_o, e := open_o.Stat()
if e != nil {
return buf_o, e
}
buf_o.Grow(bytes.MinRead + int(stat_o.Size()))
and the memory went down to 107 MB, basically the same as the first example.

Binary string to unicode

I'm not 100% sure why my binary string to unicode isn't working..can anyone point out the issue or help me patch it? Also the reason why i chunk out the binary is that it is too large for ParseInt to handle. See the playground link below for an example.
func binToString(s []byte) string {
var counter int
chunk := make([]byte, 7)
var buf bytes.Buffer
for i := range s {
if i%8 == 0 {
counter = 0
if i, err := strconv.ParseInt(string(chunk), 2, 64); err == nil {
buf.WriteString(string(i))
}
} else {
chunk[counter] = s[i] //i know i can use modulus here too but i was testing and an counter was easier to track and test for me
counter++
}
}
return buf.String()
}
It either seems to miss a character or add an character (or two) on conversion.
Here is a playground link showing an example of the function not working as expected.
Your function could be implemented in a simpler, more efficient manner:
func binToString(s []byte) string {
output := make([]byte, len(s)/8)
for i := 0; i < len(output); i++ {
val, err := strconv.ParseInt(string(s[i*8:(i+1)*8]), 2, 64)
if err == nil {
output[i] = byte(val)
}
}
return string(output)
}
https://play.golang.org/p/Fmo7I-rN3c

How to properly wait for an event/process to finish not being the parent?

I am using GO to check if a process (not been parent) has ben terminated, basically something like the pwait command in FreeBSD but written in go.
Currently I am trying a for loop with a kill -0, but I notice that the CPU usage is very high 99% with this approach, here is the code:
package main
import (
"fmt"
"os"
"strconv"
"syscall"
"time"
)
func main() {
if len(os.Args) != 2 {
fmt.Printf("usage: %s pid", os.Args[0])
os.Exit(1)
}
pid, err := strconv.ParseInt(os.Args[1], 10, 64)
if err != nil {
panic(err)
}
process, err := os.FindProcess(int(pid))
err = process.Signal(syscall.Signal(0))
for err == nil {
err = process.Signal(syscall.Signal(0))
time.Sleep(500 * time.Millisecond)
}
fmt.Println(err)
}
Any idea of how to improve or properly implement this.
Thanks in advance.
UPDATE
Adding a sleep within the loop like suggested, helps reducing the load.
From the provided links, seems to be possible to attach to the existing pid, I will give a try PtraceAttach but don't know if this may have side effects, any idea?
As suggested I was available to use kqueue:
package main
import (
"fmt"
"log"
"os"
"strconv"
"syscall"
)
func main() {
if len(os.Args) != 2 {
fmt.Printf("usage: %s pid", os.Args[0])
os.Exit(1)
}
pid, err := strconv.ParseInt(os.Args[1], 10, 64)
if err != nil {
panic(err)
}
process, _ := os.FindProcess(int(pid))
kq, err := syscall.Kqueue()
if err != nil {
fmt.Println(err)
}
ev1 := syscall.Kevent_t{
Ident: uint64(process.Pid),
Filter: syscall.EVFILT_PROC,
Flags: syscall.EV_ADD,
Fflags: syscall.NOTE_EXIT,
Data: 0,
Udata: nil,
}
for {
events := make([]syscall.Kevent_t, 1)
n, err := syscall.Kevent(kq, []syscall.Kevent_t{ev1}, events, nil)
if err != nil {
log.Println("Error creating kevent")
}
if n > 0 {
break
}
}
fmt.Println("fin")
}
Works fine, but wondering how to implement/achieve the same on linux since I think kqueue not available on it, any ideas ?
One solution would be to use the netlink proc connector, which is a socket the kernel uses to let userspace know about different process events. The official documentation is somewhat lacking, although there are a couple of good examples in C which are probably better to read.
The main caveat to using the proc connector is the process must be run as root. If running your program as a non-root user is a requirement, you should consider other options, such as periodically polling /proc to watch for changes. Any approach which uses polling, as others have pointed out, is susceptible to a race condition if the process is terminated and another one is started with the same PID in between polls.
Anyway, to use the proc connector in Go, we will have to do some translation from C. Specifically, we need to define the proc_event and exit_proc_event structs from cn_proc.h, and the cn_msg and cb_id structs from connector.h.
// CbID corresponds to cb_id in connector.h
type CbID struct {
Idx uint32
Val uint32
}
// CnMsg corresponds to cn_msg in connector.h
type CnMsg struct {
ID CbID
Seq uint32
Ack uint32
Len uint16
Flags uint16
}
// ProcEventHeader corresponds to proc_event in cn_proc.h
type ProcEventHeader struct {
What uint32
CPU uint32
Timestamp uint64
}
// ExitProcEvent corresponds to exit_proc_event in cn_proc.h
type ExitProcEvent struct {
ProcessPid uint32
ProcessTgid uint32
ExitCode uint32
ExitSignal uint32
}
We also need to make a netlink socket and call bind.
sock, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_DGRAM, unix.NETLINK_CONNECTOR)
if err != nil {
fmt.Println("socket: %v", err)
return
}
addr := &unix.SockaddrNetlink{Family: unix.AF_NETLINK, Groups: C.CN_IDX_PROC, Pid: uint32(os.Getpid())}
err = unix.Bind(sock, addr)
if err != nil {
fmt.Printf("bind: %v\n", err)
return
}
Next, we have to send the PROC_CN_MCAST_LISTEN message to the kernel to let it know we want to receive events. We can import this directly from C, where it's defined as an enum, to save some typing, and put it in a function since we will have to call it again with PROC_CN_MCAST_IGNORE when we are done receiving data from the kernel.
// #include <linux/cn_proc.h>
// #include <linux/connector.h>
import "C"
func send(sock int, msg uint32) error {
destAddr := &unix.SockaddrNetlink{Family: unix.AF_NETLINK, Groups: C.CN_IDX_PROC, Pid: 0} // the kernel
cnMsg := CnMsg{}
header := unix.NlMsghdr{
Len: unix.NLMSG_HDRLEN + uint32(binary.Size(cnMsg) + binary.Size(msg)),
Type: uint16(unix.NLMSG_DONE),
Flags: 0,
Seq: 1,
Pid: uint32(unix.Getpid()),
}
msg.ID = CbID{Idx: C.CN_IDX_PROC, Val: C.CN_VAL_PROC}
msg.Len = uint16(binary.Size(msg))
msg.Ack = 0
msg.Seq = 1
buf := bytes.NewBuffer(make([]byte, 0, header.Len))
binary.Write(buf, binary.LittleEndian, header)
binary.Write(buf, binary.LittleEndian, cnMsg)
binary.Write(buf, binary.LittleEndian, msg)
return unix.Sendto(sock, buf.Bytes(), 0, destAddr)
}
After we let the kernel know we're ready to receive events, we can receive them on the socket we're created. Once we receive them, we need to parse them, and check for relevant data. We only care about messages that meet the following criteria:
Come from the kernel
Have a header type of NLMSG_DONE
Have a proc_event_header.what value of PROC_EVENT_EXIT
Match our PID
If they meet these criteria, we can extract the relevant process information into a proc_event_exit struct, which contains the PID of the process.
for {
p := make([]byte, 1024)
nr, from, err := unix.Recvfrom(sock, p, 0)
if sockaddrNl, ok := from.(*unix.SockaddrNetlink); !ok || sockaddrNl.Pid != 0 {
continue
}
if err != nil {
fmt.Printf("Recvfrom: %v\n", err)
continue
}
if nr < unix.NLMSG_HDRLEN {
continue
}
// the sys/unix package doesn't include the ParseNetlinkMessage function
nlmessages, err := syscall.ParseNetlinkMessage(p[:nr])
if err != nil {
fmt.Printf("ParseNetlinkMessage: %v\n", err)
continue
}
for _, m := range(nlmessages) {
if m.Header.Type == unix.NLMSG_DONE {
buf := bytes.NewBuffer(m.Data)
msg := &CnMsg{}
hdr := &ProcEventHeader{}
binary.Read(buf, binary.LittleEndian, msg)
binary.Read(buf, binary.LittleEndian, hdr)
if hdr.What == C.PROC_EVENT_EXIT {
event := &ExitProcEvent{}
binary.Read(buf, binary.LittleEndian, event)
pid := int(event.ProcessTgid)
fmt.Printf("%d just exited.\n", pid)
}
}
}
}
A full code example is here.

program only prints last string of input file

i am trying to create a simple program to read lines from a text file and print them out to the console in golang. I spent lots of time going over my code and I simply can't understand why only the last line is being printed out to the screen. can anyone tell me where I am going wrong here? Everything here should compile and run.
package main
import (
"bufio"
"fmt"
"os"
)
func Readln(r *bufio.Reader) (string, error) {
var (
isPrefix bool = true
err error = nil
line, ln []byte
)
for isPrefix && err == nil {
line, isPrefix, err = r.ReadLine()
ln = append(ln, line...)
}
return string(ln), err
}
func main() {
f, err := os.Open("tickers.txt")
if err != nil {
fmt.Printf("error opening file: %v\n", err)
os.Exit(1)
}
r := bufio.NewReader(f)
s, e := Readln(r)
for e == nil {
fmt.Println(s)
s, e = Readln(r)
}
}
I therefore suspect that the problem is in your tickers.txt file line endings. The docs for ReadLine() also indicate that for most situations a Scanner is more suitable.
The following SO question has some useful information for alternative implementations: reading file line by line in go
I then used the example in the above question to re-implement your main function as follows:
f, err := os.Open("tickers.txt")
if err != nil {
fmt.Printf("error opening file: %v\n", err)
os.Exit(1)
}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
fmt.Println(scanner.Text())
}
if err := scanner.Err(); err != nil {
fmt.Println(err)
}

Resources