Skip to content

Commit

Permalink
perf: use CircularFifoQueue
Browse files Browse the repository at this point in the history
  • Loading branch information
cnzakii committed Jun 21, 2024
1 parent d0f3b7b commit 3992527
Show file tree
Hide file tree
Showing 11 changed files with 207 additions and 209 deletions.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.eventmesh.connector.http.common;

import org.apache.commons.collections4.queue.CircularFifoQueue;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;


/**
* SynchronizedCircularFifoQueue is a synchronized version of CircularFifoQueue.
*/
public class SynchronizedCircularFifoQueue<E> extends CircularFifoQueue<E> {

/**
* <p>Default constructor. capacity = 32</p>
*/
public SynchronizedCircularFifoQueue() {
super();
}

public SynchronizedCircularFifoQueue(Collection<? extends E> coll) {
super(coll);
}

public SynchronizedCircularFifoQueue(int size) {
super(size);
}

@Override
public synchronized boolean add(E element) {
return super.add(element);
}

@Override
public synchronized void clear() {
super.clear();
}

@Override
public synchronized E element() {
return super.element();
}

@Override
public synchronized E get(int index) {
return super.get(index);
}

@Override
public synchronized boolean isAtFullCapacity() {
return super.isAtFullCapacity();
}

@Override
public synchronized boolean isEmpty() {
return super.isEmpty();
}

@Override
public synchronized boolean isFull() {
return super.isFull();
}

@Override
public synchronized int maxSize() {
return super.maxSize();
}

@Override
public synchronized boolean offer(E element) {
return super.offer(element);
}

@Override
public synchronized E peek() {
return super.peek();
}

@Override
public synchronized E poll() {
return super.poll();
}

@Override
public synchronized E remove() {
return super.remove();
}

@Override
public synchronized int size() {
return super.size();
}

/**
* <p>Fetch a range of elements from the queue.</p>
*
* @param start start index
* @param end end index
* @param removed whether to remove the elements from the queue
* @return list of elements
*/
public synchronized List<E> fetchRange(int start, int end, boolean removed) {

if (start < 0 || end > this.size() || start > end) {
throw new IllegalArgumentException("Invalid range");
}

Iterator<E> iterator = this.iterator();
List<E> items = new ArrayList<>(end - start);

int count = 0;
while (iterator.hasNext() && count < end) {
E item = iterator.next();
if (item != null && count >= start) {
// Add the element to the list
items.add(item);
if (removed) {
// Remove the element from the queue
iterator.remove();
}
}
count++;
}
return items;

}


}
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public Future<HttpResponse<Buffer>> deliver(URI url, HttpConnectRecord httpConne
// convert the result to an HttpExportRecord
HttpExportRecord exportRecord = covertToExportRecord(httpConnectRecord, event, event.getResult(), event.getException(), url, id);
// add the data to the queue
((WebhookHttpSinkHandler) sinkHandler).getReceivedDataQueue().offerWithReplace(exportRecord);
((WebhookHttpSinkHandler) sinkHandler).getReceivedDataQueue().offer(exportRecord);
}
})
.onRetry(event -> {
Expand All @@ -144,7 +144,7 @@ public Future<HttpResponse<Buffer>> deliver(URI url, HttpConnectRecord httpConne
if (connectorConfig.getWebhookConfig().isActivate()) {
HttpExportRecord exportRecord =
covertToExportRecord(httpConnectRecord, event, event.getLastResult(), event.getLastException(), url, id);
((WebhookHttpSinkHandler) sinkHandler).getReceivedDataQueue().offerWithReplace(exportRecord);
((WebhookHttpSinkHandler) sinkHandler).getReceivedDataQueue().offer(exportRecord);
}
// update the HttpConnectRecord
httpConnectRecord.setTime(LocalDateTime.now().toString());
Expand All @@ -159,7 +159,7 @@ public Future<HttpResponse<Buffer>> deliver(URI url, HttpConnectRecord httpConne
}
if (connectorConfig.getWebhookConfig().isActivate()) {
HttpExportRecord exportRecord = covertToExportRecord(httpConnectRecord, event, event.getResult(), event.getException(), url, id);
((WebhookHttpSinkHandler) sinkHandler).getReceivedDataQueue().offerWithReplace(exportRecord);
((WebhookHttpSinkHandler) sinkHandler).getReceivedDataQueue().offer(exportRecord);
}
}).build();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.apache.eventmesh.connector.http.sink.handle;

import org.apache.eventmesh.common.exception.EventMeshException;
import org.apache.eventmesh.connector.http.common.BoundedConcurrentQueue;
import org.apache.eventmesh.connector.http.common.SynchronizedCircularFifoQueue;
import org.apache.eventmesh.connector.http.sink.config.HttpWebhookConfig;
import org.apache.eventmesh.connector.http.sink.config.SinkConnectorConfig;
import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord;
Expand Down Expand Up @@ -70,19 +70,19 @@ public class WebhookHttpSinkHandler extends CommonHttpSinkHandler {
private HttpServer exportServer;

// store the received data, when webhook is enabled
private final BoundedConcurrentQueue<HttpExportRecord> receivedDataQueue;
private final SynchronizedCircularFifoQueue<HttpExportRecord> receivedDataQueue;

public WebhookHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) {
super(sinkConnectorConfig);
this.sinkConnectorConfig = sinkConnectorConfig;
this.webhookConfig = sinkConnectorConfig.getWebhookConfig();
int maxQueueSize = this.webhookConfig.getMaxStorageSize();
this.receivedDataQueue = new BoundedConcurrentQueue<>(maxQueueSize);
this.receivedDataQueue = new SynchronizedCircularFifoQueue<>(maxQueueSize);
// init the export server
doInitExportServer();
}

public BoundedConcurrentQueue<HttpExportRecord> getReceivedDataQueue() {
public SynchronizedCircularFifoQueue<HttpExportRecord> getReceivedDataQueue() {
return receivedDataQueue;
}

Expand Down Expand Up @@ -129,7 +129,7 @@ private void doInitExportServer() {
int pageNum = StringUtils.isBlank(pageNumStr) ? 1 : Integer.parseInt(pageNumStr);
int pageSize = Integer.parseInt(pageSizeStr);

if (receivedDataQueue.getCurrSize() == 0) {
if (receivedDataQueue.isEmpty()) {
ctx.response()
.putHeader(HttpHeaders.CONTENT_TYPE, "application/json; charset=utf-8")
.setStatusCode(HttpResponseStatus.NO_CONTENT.code())
Expand Down Expand Up @@ -236,7 +236,7 @@ public Future<HttpResponse<Buffer>> deliver(URI url, HttpConnectRecord httpConne
// create ExportRecord
HttpExportRecord exportRecord = new HttpExportRecord(httpExportMetadata, arr.succeeded() ? arr.result().bodyAsString() : null);
// add the data to the queue
receivedDataQueue.offerWithReplace(exportRecord);
receivedDataQueue.offer(exportRecord);
});
}

Expand Down
Loading

0 comments on commit 3992527

Please sign in to comment.