Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(logfwd): support forwarding logs to Loki #267

Merged
merged 39 commits into from
Sep 25, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
f56d4ad
Implement Loki client
barrettj12 Jul 10, 2023
8ff49d8
update README
barrettj12 Aug 28, 2023
72942ad
don't need to allocate for errors.As
barrettj12 Aug 28, 2023
e91b3d9
remove clienterr package
barrettj12 Aug 29, 2023
4d8e66f
don't drop logs on 429 / 5xx
barrettj12 Aug 29, 2023
a702d37
remove retryAfter from loki.Client
barrettj12 Aug 29, 2023
a8b0c3e
revert changes to .golangci.yml
barrettj12 Aug 29, 2023
2c427e4
rename gatherer timer -> flushTimer
barrettj12 Aug 29, 2023
2d62565
rename SetRequestTimeout -> FakeRequestTimeout
barrettj12 Aug 29, 2023
8a77df6
[README] better names for example targets
barrettj12 Aug 29, 2023
0af606a
empty buffer inside handleServerResponse
barrettj12 Aug 30, 2023
7e00ccd
fix errFromResponse
barrettj12 Aug 30, 2023
5d21565
elaborate on retry comments
barrettj12 Aug 30, 2023
2b78abe
address Gustavo's review comments
barrettj12 Sep 19, 2023
0142e7e
rejig loki error handling
barrettj12 Sep 19, 2023
79187af
if-else for debug logging error
barrettj12 Sep 19, 2023
1c694b3
success is only 200 or 204
barrettj12 Sep 20, 2023
0ea5f34
pull log-counting / flushing logic into gatherer
barrettj12 Sep 20, 2023
f7ef7ff
store entries in slice
barrettj12 Sep 20, 2023
f9fea04
fix up the tests
barrettj12 Sep 20, 2023
892c037
rename AddLog to Add
barrettj12 Sep 20, 2023
bce2a86
continue stmt after flush
barrettj12 Sep 20, 2023
2dbbfb6
count # of written logs in gatherer
barrettj12 Sep 20, 2023
9b72978
reallocate buffer periodically to avoid memory leaks
barrettj12 Sep 20, 2023
fabf8a5
add test for retry/truncate logic
barrettj12 Sep 20, 2023
1c0ed80
fix gathererSuite.TestRetryLoki
barrettj12 Sep 21, 2023
8ea3930
fix imports
barrettj12 Sep 21, 2023
6a733c0
increase timeouts to improve test reliability
barrettj12 Sep 21, 2023
3c6e172
remove unnecessary comment
barrettj12 Sep 21, 2023
4dab1dd
bump test timeouts to 1 sec
barrettj12 Sep 21, 2023
084544b
use json.Compact in tests
barrettj12 Sep 21, 2023
a5ab88c
loki test: handler doesn't need to be pointer
barrettj12 Sep 21, 2023
c4e153c
Address Gustavo's review comments
barrettj12 Sep 22, 2023
164f9ad
Rename logGathererArgs -> *logGathererOptions
barrettj12 Sep 22, 2023
c6b27b0
tweak 4xx comment
barrettj12 Sep 22, 2023
e80cfcd
when truncating, zero the element to allow GC
barrettj12 Sep 22, 2023
159bbe9
reuse same buffer instead of reallocating
barrettj12 Sep 25, 2023
a94d8ff
add test for buffer recycling
barrettj12 Sep 25, 2023
20c76b2
Address Ben's comments on testing
barrettj12 Sep 25, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions internals/overlord/logstate/loki/export_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright (c) 2023 Canonical Ltd
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License version 3 as
// published by the Free Software Foundation.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package loki

type LokiEntryWithService = lokiEntryWithService

func GetBuffer(c *Client) []LokiEntryWithService {
return c.buffer
}

func GetMessage(e LokiEntryWithService) string {
return e.entry[1]
}
59 changes: 59 additions & 0 deletions internals/overlord/logstate/loki/loki_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,65 @@ func (*suite) TestServerTimeout(c *C) {
c.Assert(err, ErrorMatches, ".*context deadline exceeded.*")
}

func (*suite) TestBufferFull(c *C) {
client := loki.NewClientWithOptions(
&plan.LogTarget{
Name: "tgt1",
Location: "fake",
},
&loki.ClientOptions{
MaxRequestEntries: 3,
},
)

checkBuffer := func(expected []any) {
checkBuffer(c, client, expected)
barrettj12 marked this conversation as resolved.
Show resolved Hide resolved
}
addEntry := func(s string) {
err := client.Add(servicelog.Entry{Message: s})
c.Assert(err, IsNil)
}

checkBuffer([]any{nil, nil, nil, nil, nil, nil})
addEntry("1")
checkBuffer([]any{"1", nil, nil, nil, nil, nil})
addEntry("2")
checkBuffer([]any{"1", "2", nil, nil, nil, nil})
addEntry("3")
checkBuffer([]any{"1", "2", "3", nil, nil, nil})
addEntry("4")
checkBuffer([]any{nil, "2", "3", "4", nil, nil})
barrettj12 marked this conversation as resolved.
Show resolved Hide resolved
addEntry("5")
checkBuffer([]any{nil, nil, "3", "4", "5", nil})
addEntry("6")
checkBuffer([]any{nil, nil, nil, "4", "5", "6"})
addEntry("7")
checkBuffer([]any{"5", "6", "7", nil, nil, nil})
}

// Check if the client's buffer is as expected
func checkBuffer(c *C, client *loki.Client, expected []any) {
buffer := loki.GetBuffer(client)
if len(buffer) != len(expected) {
c.Fatalf("buffer length is %v, expected %v", len(buffer), len(expected))
}

for i := range expected {
// 'nil' means c.buffer[i] should be zero
if expected[i] == nil {
c.Assert(buffer[i], DeepEquals, loki.LokiEntryWithService{},
Commentf("buffer[%d] should be zero, obtained %v", i, buffer[i]))

} else if msg, ok := expected[i].(string); ok {
barrettj12 marked this conversation as resolved.
Show resolved Hide resolved
// Check buffer message matches
c.Assert(loki.GetMessage(buffer[i]), Equals, msg)

} else {
c.Fatalf("invalid value expected[%d] = %v %T", i, expected[i], expected[i])
}
}
}

// Strips all extraneous whitespace from JSON
func compactJSON(s string) []byte {
var buf bytes.Buffer
Expand Down
Loading