From 88530a7944a47ea7619ee42384d597179405bf54 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Fri, 8 Nov 2019 12:34:36 -0800 Subject: [PATCH 1/2] if someone logs a line longer than 65*1024 bytes before it contains a newline, the default ScanLines func in bufio.Scanner can't handle it. Copy that func and add a conditional to clear buffer and continue scanning in this situation. --- log.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/log.go b/log.go index 1a6722235..021de1f4e 100644 --- a/log.go +++ b/log.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "bytes" "fmt" "io" "os" @@ -30,6 +31,8 @@ func logOutput() (logOutput io.Writer, err error) { // calls. r, w := io.Pipe() scanner := bufio.NewScanner(r) + scanner.Split(ScanLinesSmallerThanBuffer) + go func(scanner *bufio.Scanner) { for scanner.Scan() { if strings.Contains(scanner.Text(), "ui:") { @@ -47,3 +50,39 @@ func logOutput() (logOutput io.Writer, err error) { return } + +// The below functions come from bufio.Scanner with a small tweak, to fix an +// edgecase where the default ScanFunc fails: sometimes, if someone tries to +// log a line that is longer than 64*1024 bytes long before it contains a +// newline, the ScanLine will continue to return, requesting more data from the +// buffer, which can't increase in size anymore, causing a hang. +func dropCR(data []byte) []byte { + if len(data) > 0 && data[len(data)-1] == '\r' { + return data[0 : len(data)-1] + } + return data +} + +func ScanLinesSmallerThanBuffer(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + if i := bytes.IndexByte(data, '\n'); i >= 0 { + // We have a full newline-terminated line. + return i + 1, dropCR(data[0:i]), nil + } + // If we're at EOF, we have a final, non-terminated line. Return it. + if atEOF { + return len(data), dropCR(data), nil + } + + // Our tweak: + // Buffer is full, so we can't get more data. Just return what we have as + // its own token so we can keep going, even though there's no newline. + if len(data)+1 >= bufio.MaxScanTokenSize { + return len(data), data[0 : len(data)-1], nil + } + + // Request more data. + return 0, nil, nil +} From 60cf4f60140a89a2feee3072bd93f7faeaae5bc0 Mon Sep 17 00:00:00 2001 From: Megan Marsh Date: Mon, 11 Nov 2019 09:19:35 -0800 Subject: [PATCH 2/2] log errors in scanner and manually close logging pipe if we hit an error in the scanner --- log.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/log.go b/log.go index 021de1f4e..ff12c94c0 100644 --- a/log.go +++ b/log.go @@ -43,6 +43,10 @@ func logOutput() (logOutput io.Writer, err error) { } os.Stderr.WriteString(fmt.Sprint(scanner.Text() + "\n")) } + if err := scanner.Err(); err != nil { + os.Stderr.WriteString(err.Error()) + w.Close() + } }(scanner) logOutput = w }