Go program for indexing file systems - linux

I'm coding a program to map very big file systems (over 100TB). It is using fd to list files/directories. In the end, the program is supposed to make text files (that will be merged later) containing paths of everything on file system from the point I executed it. Problem is that it keeps cycling and printing the same output to the file, so I suppose mistake is somewhere in declaration/rewriting of some variables. What do you think?
package main
import (
"strconv"
"os/exec"
"bufio"
"strings"
"fmt"
"bytes"
"log"
"os"
)
// declaring variables
var count int
var start_dir string
var cmd *exec.Cmd
var sizeof_dir_array int
var counter int
var index_output string
var dirs string
var out bytes.Buffer
var stderr bytes.Buffer
var byte_out []byte
var err error
var err1 error
var dir_array []string
func find_indexes(starting_directory string) {
start_dir = starting_directory
sizeof_dir_array = 0
counter = 0
index_output = ""
// deciding to use start_dir or not
if start_dir != "" {
// declaring byte output that contain file indexes and issigning it to index_output variable
byte_out, err = exec.Command("fd", "-d", "1", "-a", ".", start_dir).Output()
if err != nil {
log.Fatal(err)
}
index_output = string(byte_out)
// issigning string containing directories to cmd variable
cmd = exec.Command("fd", "-d", "1", "-a", "-t", "d", ".", start_dir)
}else {
byte_out, err = exec.Command("fd", "-d", "1", "-a").Output()
if err != nil {
log.Fatal(err)
}
index_output = string(byte_out)
cmd = exec.Command("fd", "-d", "1", "-a", "-t", "d")
}
// parsing from cmd to dirs variable
cmd.Stdout = &out
cmd.Stderr = &stderr
err1 = cmd.Run()
if err1 != nil {
fmt.Println(fmt.Sprint(err1) + ": " + stderr.String())
return
}
dirs = out.String()
// writing index_output to a file
file, err2 := os.Create("{your_path}/index-" + strconv.Itoa(count) + ".txt")
if err2 != nil {
log.Fatal("Cannot create file", err2)
}
count++
fmt.Fprintf(file, index_output)
file.Close()
// putting dirs from string to dir_array
scanner := bufio.NewScanner(strings.NewReader(string(dirs)))
for scanner.Scan() {
sizeof_dir_array++
}
dir_array = make([]string, sizeof_dir_array)
for scanner.Scan() {
dir_array[counter] = scanner.Text()
counter++
}
// if there are no more dirs, recursion continues
if dirs != "" {
for _, val := range dir_array {
find_indexes(val)
}
}
}
func main() {
count = 0
find_indexes("")
}

Related

How to split string two between characters

I want to split a string up between two characters( {{ and }} ).
I have an string like {{number1}} + {{number2}} > {{number3}}
and I'm looking for something that returns:
[number1, number2, number3]
You can try it with Regex:
s := "{{number1}} + {{number2}} > {{number3}}"
// Find all substrings in form {<var name>}
re := regexp.MustCompile("{[a-z]*[0-9]*[a-z]*}")
nums := re.FindAllString(s, -1)
// Remove '{' and '}' from all substrings
for i, _ := range nums {
nums[i] = strings.TrimPrefix(nums[i], "{")
nums[i] = strings.TrimSuffix(nums[i], "}")
}
fmt.Println(nums) // output: [number1 number2 number3]
You can experiment with regex here: https://regex101.com/r/kkPWAS/1
Use the regex [A-Za-z]+[0-9] and filter the alpha numeric parts of the string as string array.
package main
import (
"fmt"
"regexp"
)
func main() {
s := `{{number1}} + {{number2}} > {{number3}}`
re := regexp.MustCompile("[A-Za-z]+[0-9]")
p := re.FindAllString(s, -1)
fmt.Println(p) //[number1 number2 number3]
}
the hard way using the template parser ^^
package main
import (
"fmt"
"strings"
"text/template/parse"
)
func main() {
input := "{{number1}} + {{number2}} > {{number3}}"
out := parseit(input)
fmt.Printf("%#v\n", out)
}
func parseit(input string) (out []string) {
input = strings.Replace(input, "{{", "{{.", -1) // Force func calls to become variables.
tree, err := parse.Parse("", input, "{{", "}}")
if err != nil {
panic(err)
}
visit(tree[""].Root, func(n parse.Node) bool {
x, ok := n.(*parse.FieldNode)
if ok {
out = append(out, strings.Join(x.Ident, "."))
}
return true
})
return
}
func visit(n parse.Node, fn func(parse.Node) bool) bool {
if n == nil {
return true
}
if !fn(n) {
return false
}
if l, ok := n.(*parse.ListNode); ok {
for _, nn := range l.Nodes {
if !visit(nn, fn) {
continue
}
}
}
if l, ok := n.(*parse.RangeNode); ok {
if !visit(l.BranchNode.Pipe, fn) {
return false
}
if l.BranchNode.List != nil {
if !visit(l.BranchNode.List, fn) {
return false
}
}
if l.BranchNode.ElseList != nil {
if !visit(l.BranchNode.ElseList, fn) {
return false
}
}
}
if l, ok := n.(*parse.ActionNode); ok {
for _, c := range l.Pipe.Decl {
if !visit(c, fn) {
continue
}
}
for _, c := range l.Pipe.Cmds {
if !visit(c, fn) {
continue
}
}
}
if l, ok := n.(*parse.CommandNode); ok {
for _, a := range l.Args {
if !visit(a, fn) {
continue
}
}
}
if l, ok := n.(*parse.PipeNode); ok {
for _, a := range l.Decl {
if !visit(a, fn) {
continue
}
}
for _, a := range l.Cmds {
if !visit(a, fn) {
continue
}
}
}
return true
}
If it happens you really were manipulating template string, but fails to do so due to function calls and that you do not want to execute this input = strings.Replace(input, "{{", "{{.", -1) // Force func calls to become variables.
You can always force load a template using functions similar to
var reMissingIdent = regexp.MustCompile(`template: :[0-9]+: function "([^"]+)" not defined`)
func ParseTextTemplateAnyway(s string) (*texttemplate.Template, texttemplate.FuncMap, error) {
fn := texttemplate.FuncMap{}
for {
t, err := texttemplate.New("").Funcs(fn).Parse(s)
if err == nil {
return t, fn, err
}
s := err.Error()
res := reMissingIdent.FindAllStringSubmatch(s, -1)
if len(res) > 0 {
fn[res[0][1]] = func(s ...interface{}) string { return "" }
} else {
return t, fn, err
}
}
// return nil, nil
}
You don't need to use libraries. You can create your own function.
package main
const r1 = '{'
const r2 = '}'
func GetStrings(in string) (out []string) {
var tren string
wr := false
f := true
for _, c := range in {
if wr && c != r2 {
tren = tren + string(c)
}
if c == r1 {
f = !f
wr = f
}
if c == r2 {
wr = false
if f {
out = append(out, tren)
tren = ""
}
f = !f
}
}
return
}

I can't return string slice like I would like to. Passing just the last one

I wrote this code to get the list of the file in directory, appending the names in a slice and one by one open them, after I open a file I search for some words in the file and if found write them in a new file.
But I always get the same words in the new files and I can't figure out why
package main
import (
"bufio"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
)
const dir_to_read_path string = "path"
func main() {
start := time.Now()
temp_string_filename := ""
temp_string_filename_counter := 0
//defer list_file()
// just pass the file name
for k := range list_file() {
temp_string_filename = list_file()[temp_string_filename_counter]
if true {
k = k
}
temp_string_filename_counter++
b, err := ioutil.ReadFile(temp_string_filename)
if err != nil {
fmt.Print(err)
}
// convert content to a 'string'
str := string(b)
control_params := []string{"numpy", "grabscreen", "cv2", "time", "os", "pandas", "tqdm", "collections", "models", "random", "inception_v3", "googlenet", "shuffle", "getkeys", "tflearn", "directkeys", "statistics", "motion", "tflearn.layers.conv", "conv_2d", "max_pool_2d", "avg_pool_2d", "conv_3d", "max_pool_3d", "avg_pool_3d"}
temp_string_filename = dir_to_read_path + "output_" + temp_string_filename
fmt.Println("Writing file n. ", k)
file, err := os.Create(temp_string_filename)
if err != nil {
log.Fatal("Cannot create file", err)
}
for _, z := range isValueInList(control_params, str, list_file()) {
fmt.Fprintf(file, z)
fmt.Fprintf(file, "\n")
}
defer file.Close()
elapsed := time.Since(start)
log.Printf("Execution took %s", elapsed)
}
}
func isValueInList(list []string, file_string string, read_file []string) []string {
encountered_modules := make([]string, 0, 10)
temp_string_filename := ""
temp_string_filename_counter := 0
encountered := map[string]bool{}
result := make([]string, 0, 10)
final_result := [][]string{}
for z := range read_file {
fmt.Println("Reading file n. ", z)
temp_string_filename = read_file[temp_string_filename_counter]
f, _ := os.Open(temp_string_filename)
defer f.Close()
scanner := bufio.NewScanner(f)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
line := scanner.Text()
for _, v := range list {
if v == line {
encountered_modules = append(encountered_modules, line)
}
}
}
for v := range encountered_modules {
if encountered[encountered_modules[v]] == true {
// Do not add duplicate.
} else {
// Record this element as an encountered element.
encountered[encountered_modules[v]] = true
result = append(result, encountered_modules[v])
}
}
temp_string_filename_counter++
final_result = append(final_result, result)
}
return result
}
func list_file() []string {
files_names := make([]string, 0, 10)
files, err := ioutil.ReadDir("./")
if err != nil {
log.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f.Name(), ".txt") {
files_names = append(files_names, string(f.Name()))
}
}
return files_names
}
It's hard to be sure, since your code is difficult to read, but this looks particularly suspicious (in pseudocode),
// main
for each file in list_file() {
result = {
// isValueInList
var result
for each file in list_file() {
for each word in file {
if word in wordlist and not in result {
result = append(result, word)
}
}
}
// all the words in wordlist in any of the files
return result
}
// main
write result
}
There are other problems with your code.
Here's a more readable example (a first draft), of what you appear to be trying to do (Python modules in Python files?):
package main
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
)
var modules = map[string]bool{
"numpy": true, "grabscreen": true, "cv2": true, "time": true, "os": true, "pandas": true, "tqdm": true, "collections": true,
"models": true, "random": true, "inception_v3": true, "googlenet": true, "shuffle": true, "getkeys": true, "tflearn": true,
"directkeys": true, "statistics": true, "motion": true, "tflearn.layers.conv": true, "conv_2d": true,
"max_pool_2d": true, "avg_pool_2d": true, "conv_3d": true, "max_pool_3d": true, "avg_pool_3d": true,
}
func findWords(filename string, lexicon map[string]bool) error {
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
words := make(map[string]bool)
s := bufio.NewScanner(f)
s.Split(bufio.ScanWords)
for s.Scan() {
word := s.Text()
if _, exists := lexicon[word]; exists {
words[word] = true
}
}
if s.Err(); err != nil {
return err
}
var buf bytes.Buffer
for word := range words {
buf.WriteString(word)
buf.WriteString("\n")
}
if buf.Len() > 0 {
err := ioutil.WriteFile(filename+`.words`, buf.Bytes(), 0666)
if err != nil {
return err
}
}
return nil
}
func main() {
dir := `./`
files, err := ioutil.ReadDir(dir)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
for _, file := range files {
filename := file.Name()
if filepath.Ext(filename) != ".py" {
continue
}
findWords(filename, modules)
if err != nil {
fmt.Fprintln(os.Stderr, err)
}
}
}
There are a few mistakes in your code, so i've rewritten most of the code.
What i did :
1) open a file
2) read a line
3) compare it
4) check if the target file exists
5) if not, create it
6) if it does, append to it
7) write to it
8) close target file
9) goto 2 if there are more lines
10) goto 1 if there are more files
I tried to make it as much as readable for everybody so that everybody can understand it.
package main
import (
"bufio"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const readDir string = "./"
var startTime time.Time
func main() {
for noFile, fileName := range listFile() {
startTime = time.Now()
fileInput, err := os.Open(fileName)
if err != nil {
log.Fatal(err)
}
defer fileInput.Close()
scanner := bufio.NewScanner(fileInput)
for scanner.Scan() {
for _, targetContent := range []string{"numpy", "grabscreen", "cv2", "time", "os", "pandas", "tqdm", "collections", "models", "random", "inception_v3", "googlenet", "shuffle", "getkeys", "tflearn", "directkeys", "statistics", "motion", "tflearn.layers.conv", "conv_2d", "max_pool_2d", "avg_pool_2d", "conv_3d", "max_pool_3d", "avg_pool_3d"} {
if strings.Contains(scanner.Text(), targetContent) {
if _, err := os.Stat(readDir + "output_" + strconv.Itoa(noFile)); os.IsNotExist(err) {
fmt.Println("File : " + readDir + "output_" + strconv.Itoa(noFile) + " does not exists, creating it now!")
createFile, err := os.Create(readDir + "output_" + strconv.Itoa(noFile))
if err != nil {
panic(err)
}
createFile.Close()
}
fileOutput, err := os.OpenFile(readDir+"output_"+strconv.Itoa(noFile), os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
panic(err)
}
if _, err = fileOutput.WriteString("contains : " + targetContent + " in : " + scanner.Text() + "\n"); err != nil {
panic(err)
}
fileOutput.Close()
fmt.Println("Writing file : ", readDir+"output_"+strconv.Itoa(noFile))
fmt.Println("contains : " + targetContent + " in : " + scanner.Text())
}
}
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
log.Printf("Execution took %s", time.Since(startTime))
}
}
func listFile() []string {
filesNames := make([]string, 0, 100)
files, err := ioutil.ReadDir(readDir)
if err != nil {
log.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f.Name(), ".txt") {
fileName, err := filepath.Abs(string(f.Name()))
if err != nil {
log.Fatal(err)
}
filesNames = append(filesNames, fileName)
}
}
return filesNames
}

Golang Enter SSH Sudo Password on Prompt (or exit)

I'm trying to run a script via the SSH package in my Go program (so far I've had success).
My issue is, the script attempts to run a command with sudo if the user has sudo privileges, and this causes the bash script to pause until a password is entered by the user.
For example:
[ERROR ] Install cs-server: Checking dependencies: missing: lib32gcc1
# It attempts to install the missing dependencies with sudo but pauses here
[sudo] password for guest:
In my Go program, I have written something that looks similar to this:
// Connect to SSH and retreive session...
out, err := session.StdoutPipe()
if err != nil {
log.Fatal(err)
}
go func(out io.Reader) {
r := bufio.NewScanner(out)
for r.Scan() {
fmt.Println(r.Text())
}
}(out)
// Execute ssh command...
And I receive the exact same output as the example above, only in this case, I don't even see the line [sudo] password for guest:... it only prints up to [ERROR ] Install cs-server: Checking dependencies: missing: lib32gcc1 and pauses forever.
How can I bypass this pause? My options are to either enter the password from my Go program automatically, or end the ssh execution and just receive the output.
I managed to fix this issue by making use of the session.StdoutPipe() and session.StdinPipe(). I wrote a go routine which scans each byte and checks if the last written line starts with "[sudo] password for " and ends with ": ". It will write the password + "\n" to the session.StdinPipe() which continues execution of the script.
Here's all of the code I have for this.
package ssh
import (
"bufio"
"io"
"log"
"net"
"strings"
"golang.org/x/crypto/ssh"
)
type Connection struct {
*ssh.Client
password string
}
func Connect(addr, user, password string) (*Connection, error) {
sshConfig := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{
ssh.Password(password),
},
HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }),
}
conn, err := ssh.Dial("tcp", addr, sshConfig)
if err != nil {
return nil, err
}
return &Connection{conn, password}, nil
}
func (conn *Connection) SendCommands(cmds ...string) ([]byte, error) {
session, err := conn.NewSession()
if err != nil {
log.Fatal(err)
}
defer session.Close()
modes := ssh.TerminalModes{
ssh.ECHO: 0, // disable echoing
ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
}
err = session.RequestPty("xterm", 80, 40, modes)
if err != nil {
return []byte{}, err
}
in, err := session.StdinPipe()
if err != nil {
log.Fatal(err)
}
out, err := session.StdoutPipe()
if err != nil {
log.Fatal(err)
}
var output []byte
go func(in io.WriteCloser, out io.Reader, output *[]byte) {
var (
line string
r = bufio.NewReader(out)
)
for {
b, err := r.ReadByte()
if err != nil {
break
}
*output = append(*output, b)
if b == byte('\n') {
line = ""
continue
}
line += string(b)
if strings.HasPrefix(line, "[sudo] password for ") && strings.HasSuffix(line, ": ") {
_, err = in.Write([]byte(conn.password + "\n"))
if err != nil {
break
}
}
}
}(in, out, &output)
cmd := strings.Join(cmds, "; ")
_, err = session.Output(cmd)
if err != nil {
return []byte{}, err
}
return output, nil
}
And an example of how you could use it.
// ssh refers to the custom package above
conn, err := ssh.Connect("0.0.0.0:22", "username", "password")
if err != nil {
log.Fatal(err)
}
output, err := conn.SendCommands("sleep 2", "echo Hello!")
if err != nil {
log.Fatal(err)
}
fmt.Println(string(output))
This is an issue that output stream can't be fully captured for #acidic's code.
The updated code is as following
package main
import (
"bytes"
"fmt"
"io"
"log"
"net"
"strings"
"golang.org/x/crypto/ssh"
)
type Connection struct {
*ssh.Client
password string
}
func Connect(addr, user, password string) (*Connection, error) {
sshConfig := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{
ssh.Password(password),
},
HostKeyCallback: ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil }),
}
conn, err := ssh.Dial("tcp", addr, sshConfig)
if err != nil {
return nil, err
}
return &Connection{conn, password}, nil
}
func (conn *Connection) SendCommands(cmds string) ([]byte, error) {
session, err := conn.NewSession()
if err != nil {
log.Fatal(err)
}
defer session.Close()
modes := ssh.TerminalModes{
ssh.ECHO: 0, // disable echoing
ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
}
err = session.RequestPty("xterm", 80, 40, modes)
if err != nil {
return []byte{}, err
}
stdoutB := new(bytes.Buffer)
session.Stdout = stdoutB
in, _ := session.StdinPipe()
go func(in io.Writer, output *bytes.Buffer) {
for {
if strings.Contains(string(output.Bytes()), "[sudo] password for ") {
_, err = in.Write([]byte(conn.password + "\n"))
if err != nil {
break
}
fmt.Println("put the password --- end .")
break
}
}
}(in, stdoutB)
err = session.Run(cmds)
if err != nil {
return []byte{}, err
}
return stdoutB.Bytes(), nil
}
func main() {
// ssh refers to the custom package above
conn, err := Connect("0.0.0.0:22", "username", "password")
if err != nil {
log.Fatal(err)
}
output, err := conn.SendCommands("sudo docker ps")
if err != nil {
log.Fatal(err)
}
fmt.Println(string(output))
}
A work around is converting sudo [cmd] to echo [password] | sudo -S [cmd], it is not good, but working for me.
Another workaround if you dont want to use ssh library is to make a pseudo terminal using pty library. An extremely simple example as above
import (
"io"
"os"
"os/exec"
"time"
"github.com/creack/pty"
)
func main() {
c := exec.Command("ssh", "<user>#<IP>")
f, err := pty.Start(c)
if err != nil {
panic(err)
}
time.Sleep(2 * time.Second)
f.Write([]byte("1234\n"))
io.Copy(os.Stdout, f)
}

Is it possible to include an external file as a string constant in go?

I always wished it was possible to do something like this in C++:
const std::string fragmentShader = "
#include "shader.frag"
";
Obviously that doesn't work, and there is no way to do it in C++. But it is possible in go? i.e.
const fragmentShader string = `
<insert contents of shader.frag at compile-time>
`
The motivation should be obvious!
Not really, but here are a couple options:
Just take the contents of the file and put them in your source code:
const fragmentShaderFile = `contents of shader.frag`
You can embed assets with this tool: https://github.com/jteeuwen/go-bindata, but you'll have to re-embed the file anytime it's changed. (One could automate this as a git pre-commit hook though)
This is not possible in pure Go. You could however write a program that reads a file and creates a Go file from it, such as this:
package main
import "flag"
import "os"
import "fmt"
import "bufio"
import "io"
var (
packageName = flag.String("p", "main", "package name")
outFile = flag.String("o", "-", "output file. Defaults to stdout")
varName = flag.String("v", "file", "variable name")
)
const (
header = "package %s\n\nvar %s = [...]byte{\n"
trailer = "}\n"
)
func main() {
flag.Parse()
if len(flag.Args()) != 1 {
fmt.Fprintln(os.Stderr, "Please provide exactly one file name")
os.Exit(1)
}
var inF, outF *os.File
if *outFile == "-" {
outF = os.Stdout
} else {
var err error
outF, err = os.Create(*outFile)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot create %s: %v\n", *outFile, err)
os.Exit(1)
}
}
inF, err := os.Open(flag.Args()[0])
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot open %s: %v\n", flag.Args()[0], err)
os.Exit(1)
}
in, out := bufio.NewReader(inF), bufio.NewWriter(outF)
fmt.Fprintf(out, header, *packageName, *varName)
buf := make([]byte, 16)
var n int
for n, err = io.ReadFull(in, buf); n > 0; n, err = io.ReadFull(in, buf) {
out.WriteRune('\t')
for i := 0; i < n-1; i++ {
fmt.Fprintf(out, "%#02x, ", buf[i])
}
fmt.Fprintf(out, "%#02x,\n", buf[n-1])
}
out.WriteString(trailer)
out.Flush()
if err != io.EOF {
fmt.Fprintf(os.Stderr, "An error occured while reading from %s: %v\n", flag.Args()[0], err)
os.Exit(1)
}
}

Reading from reader until a string is reached

I am trying to write a function to keep reading from a buffered reader until I hit a certain string, then to stop reading and return everything read prior to that string.
In other words, I want to do the same thing as reader.ReadString() does, except taking a string instead of a single byte.
For instance:
mydata, err := reader.ReadString("\r\n.\r\n") //obviously will not compile
How can I do this?
Thanks in advance,
Twichy
Amendment 1: Previous attempt
Here is my previous attempt; its badly written and doesnt work but hopefully it demonstrates what I am trying to do.
func readDotData(reader *bufio.Reader)(string, error){
delims := []byte{ '\r', '\n', '.', '\r', '\n'}
curpos := 0
var buffer []byte
for {
curpos = 0
data, err := reader.ReadSlice(delims[0])
if err!=nil{ return "", err }
buffer = append(buffer, data...)
for {
curpos++
b, err := reader.ReadByte()
if err!=nil{ return "", err }
if b!=delims[curpos]{
for curpos >= 0{
buffer = append(buffer, delims[curpos])
curpos--
}
break
}
if curpos == len(delims){
return string(buffer[len(buffer)-1:]), nil
}
}
}
panic("unreachable")
}
package main
import (
"bytes"
"fmt"
"log"
)
type reader interface {
ReadString(delim byte) (line string, err error)
}
func read(r reader, delim []byte) (line []byte, err error) {
for {
s := ""
s, err = r.ReadString(delim[len(delim)-1])
if err != nil {
return
}
line = append(line, []byte(s)...)
if bytes.HasSuffix(line, delim) {
return line[:len(line)-len(delim)], nil
}
}
}
func main() {
src := bytes.NewBufferString("123deli456elim789delimABCdelimDEF")
for {
b, err := read(src, []byte("delim"))
if err != nil {
log.Fatal(err)
}
fmt.Printf("%q\n", b)
}
}
Playground
Output:
"123deli456elim789"
"ABC"
2009/11/10 23:00:00 EOF
http://play.golang.org/p/BpA5pOc-Rn
package main
import (
"bytes"
"fmt"
)
func main() {
b := bytes.NewBuffer([]byte("Hello, playground!\r\n.\r\nIrrelevant trailer."))
c := make([]byte, 0, b.Len())
for {
p := b.Bytes()
if bytes.Equal(p[:5], []byte("\r\n.\r\n")) {
fmt.Println(string(c))
return
}
c = append(c, b.Next(1)...)
}
}
For example,
package main
import (
"bufio"
"bytes"
"fmt"
"strings"
)
var delim = []byte{'\r', '\n', '.', '\r', '\n'}
func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
for i := 0; i+len(delim) <= len(data); {
j := i + bytes.IndexByte(data[i:], delim[0])
if j < i {
break
}
if bytes.Equal(data[j+1:j+len(delim)], delim[1:]) {
// We have a full delim-terminated line.
return j + len(delim), data[0:j], nil
}
i = j + 1
}
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), data, nil
}
// Request more data.
return 0, nil, nil
}
func main() {
delims := string(delim)
input := "1234" + delims + "5678" + delims + "1234567901234567890" + delims
scanner := bufio.NewScanner(strings.NewReader(input))
scanner.Split(ScanLines)
for scanner.Scan() {
fmt.Printf("%s\n", scanner.Text())
}
if err := scanner.Err(); err != nil {
fmt.Printf("Invalid input: %s", err)
}
}
Output:
1234
5678
1234567901234567890
Because you have the same byte in the string, you can do it as below:
func readWithEnd(reader *bufio.Reader) ([]byte, error) {
message, err := reader.ReadBytes('#')
if err != nil {
return nil, err
}
a1, err := reader.ReadByte()
if err != nil {
return nil, err
}
message = append(message, a1)
if a1 != '\t' {
message2, err := readWithEnd(reader)
if err != nil {
return nil, err
}
ret := append(message, message2...)
return ret, nil
}
a2, err := reader.ReadByte()
if err != nil {
return nil, err
}
message = append(message, a2)
if a2 != '#' {
message2, err := readWithEnd(reader)
if err != nil {
return nil, err
}
ret := append(message, message2...)
return ret, nil
}
return message, nil
}
This is the sample that can recognize the "#\t#" in TCP connection

Resources