Skip to content

BUGFIX: Make join operation thread safe #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
prasadlohakpure merged 2 commits into from
Feb 24, 2026
Merged

BUGFIX: Make join operation thread safe #46

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8e3ecb5
DATA-7369: Make join operation thread safe
prasadlohakpure Feb 23, 2026
d8744e8
Merge remote-tracking branch 'origin/main' into join_thread_safe
prasadlohakpure Feb 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 12 additions & 13 deletions internal/pkg/pipeline/task/join/join.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@ type join struct {
Number int `yaml:"number,omitempty" json:"number,omitempty"`
Duration duration.Duration `yaml:"duration,omitempty" json:"duration,omitempty"`
Delimiter string `yaml:"delimiter,omitempty" json:"delimiter,omitempty"`
buffer []*record.Record
}

func New() (task.Task, error) {
return &join{
buffer: make([]*record.Record, 0, defaultBufferSize),
Delimiter: defaultDelimiter,
}, nil
}
Expand All @@ -49,6 +47,7 @@ func (j *join) Run(input <-chan *record.Record, output chan<- *record.Record) er
totalSize := 0
var ticker *time.Ticker
var tickerCh <-chan time.Time
buffer := make([]*record.Record, 0, defaultBufferSize)

if j.Duration > 0 {
ticker = time.NewTicker(time.Duration(j.Duration))
Expand All @@ -63,40 +62,40 @@ func (j *join) Run(input <-chan *record.Record, output chan<- *record.Record) er
r, ok := j.GetRecord(input)
if !ok {
// Input channel closed, send any remaining records
j.flushBuffer(output)
j.flushBuffer(&buffer, output)
return nil
}

j.buffer = append(j.buffer, r)
buffer = append(buffer, r)
totalSize += len(r.Data)

// Check if we've reached the size or number limit
if (j.Size > 0 && totalSize >= j.Size) || (j.Number > 0 && len(j.buffer) >= j.Number) {
j.flushBuffer(output)
if (j.Size > 0 && totalSize >= j.Size) || (j.Number > 0 && len(buffer) >= j.Number) {
j.flushBuffer(&buffer, output)
totalSize = 0
}

case <-tickerCh:
// Timer expired, send any buffered records
j.flushBuffer(output)
j.flushBuffer(&buffer, output)
totalSize = 0
}
}
}

// flushBuffer sends any buffered records and resets the buffer
func (j *join) flushBuffer(output chan<- *record.Record) {
if len(j.buffer) > 0 {
j.sendJoinedRecords(output)
j.buffer = j.buffer[:0]
func (j *join) flushBuffer(buffer *[]*record.Record, output chan<- *record.Record) {
if len(*buffer) > 0 {
j.sendJoinedRecords(*buffer, output)
*buffer = (*buffer)[:0]
}
}

func (j *join) sendJoinedRecords(output chan<- *record.Record) {
func (j *join) sendJoinedRecords(buffer []*record.Record, output chan<- *record.Record) {

// Join all data with the specified delimiter
var joinedData strings.Builder
for i, r := range j.buffer {
for i, r := range buffer {
if i > 0 {
joinedData.WriteString(j.Delimiter)
}
Expand Down