Skip to content

Commit

Permalink
Added support for SstFileManager to RocksJava
Browse files Browse the repository at this point in the history
Summary: Closes facebook#3666

Differential Revision: D7457634

Pulled By: sagar0

fbshipit-source-id: 47741e2ee66e9255c580f4e38cfb86b284c27c2f
  • Loading branch information
adamretter authored and facebook-github-bot committed Apr 7, 2018
1 parent 74767de commit ca87aef
Show file tree
Hide file tree
Showing 14 changed files with 615 additions and 0 deletions.
1 change: 1 addition & 0 deletions HISTORY.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

### Java API Changes
* Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances.
* Added SstFileManager to the Java API to allow managing SST files across DB instances.

## 5.13.0 (3/20/2018)
### Public API Change
Expand Down
3 changes: 3 additions & 0 deletions java/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ set(JNI_NATIVE_SOURCES
rocksjni/rocksjni.cc
rocksjni/slice.cc
rocksjni/snapshot.cc
rocksjni/sst_file_manager.cc
rocksjni/sst_file_writerjni.cc
rocksjni/statistics.cc
rocksjni/statisticsjni.cc
Expand Down Expand Up @@ -111,6 +112,7 @@ set(NATIVE_JAVA_CLASSES
org.rocksdb.Slice
org.rocksdb.Snapshot
org.rocksdb.SnapshotTest
org.rocksdb.SstFileManager
org.rocksdb.SstFileWriter
org.rocksdb.Statistics
org.rocksdb.StringAppendOperator
Expand Down Expand Up @@ -232,6 +234,7 @@ add_jar(
src/main/java/org/rocksdb/SkipListMemTableConfig.java
src/main/java/org/rocksdb/Slice.java
src/main/java/org/rocksdb/Snapshot.java
src/main/java/org/rocksdb/SstFileManager.java
src/main/java/org/rocksdb/SstFileWriter.java
src/main/java/org/rocksdb/Statistics.java
src/main/java/org/rocksdb/StatsLevel.java
Expand Down
2 changes: 2 additions & 0 deletions java/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractCompactionFilter\
org.rocksdb.RocksMemEnv\
org.rocksdb.SkipListMemTableConfig\
org.rocksdb.Slice\
org.rocksdb.SstFileManager\
org.rocksdb.SstFileWriter\
org.rocksdb.Statistics\
org.rocksdb.Transaction\
Expand Down Expand Up @@ -132,6 +133,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\
org.rocksdb.util.SizeUnitTest\
org.rocksdb.SliceTest\
org.rocksdb.SnapshotTest\
org.rocksdb.SstFileManagerTest\
org.rocksdb.SstFileWriterTest\
org.rocksdb.TransactionTest\
org.rocksdb.TransactionDBTest\
Expand Down
28 changes: 28 additions & 0 deletions java/rocksjni/options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -895,6 +895,20 @@ void Java_org_rocksdb_Options_setRateLimiter(
rate_limiter = *pRateLimiter;
}

/*
* Class: org_rocksdb_Options
* Method: setSstFileManager
* Signature: (JJ)V
*/
void Java_org_rocksdb_Options_setSstFileManager(
JNIEnv* env, jobject job, jlong jhandle,
jlong jsst_file_manager_handle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jsst_file_manager_handle);
reinterpret_cast<rocksdb::Options*>(jhandle)->
sst_file_manager = *sptr_sst_file_manager;
}

/*
* Class: org_rocksdb_Options
* Method: setLogger
Expand Down Expand Up @@ -4399,6 +4413,20 @@ void Java_org_rocksdb_DBOptions_setRateLimiter(
reinterpret_cast<rocksdb::DBOptions*>(jhandle)->rate_limiter = *pRateLimiter;
}

/*
* Class: org_rocksdb_DBOptions
* Method: setSstFileManager
* Signature: (JJ)V
*/
void Java_org_rocksdb_DBOptions_setSstFileManager(
JNIEnv* env, jobject jobj, jlong jhandle,
jlong jsst_file_manager_handle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jsst_file_manager_handle);
reinterpret_cast<rocksdb::DBOptions*>(jhandle)->
sst_file_manager = *sptr_sst_file_manager;
}

/*
* Class: org_rocksdb_DBOptions
* Method: setLogger
Expand Down
217 changes: 217 additions & 0 deletions java/rocksjni/sst_file_manager.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// This file implements the "bridge" between Java and C++ and enables
// calling C++ rocksdb::SstFileManager methods
// from Java side.

#include <jni.h>
#include <memory>

#include "include/org_rocksdb_SstFileManager.h"
#include "rocksdb/sst_file_manager.h"
#include "rocksjni/portal.h"

/*
* Class: org_rocksdb_SstFileManager
* Method: newSstFileManager
* Signature: (JJJDJ)J
*/
jlong Java_org_rocksdb_SstFileManager_newSstFileManager(
JNIEnv* jnienv, jclass jcls, jlong jenv_handle, jlong jlogger_handle,
jlong jrate_bytes, jdouble jmax_trash_db_ratio,
jlong jmax_delete_chunk_bytes) {

auto* env = reinterpret_cast<rocksdb::Env*>(jenv_handle);
rocksdb::Status s;
rocksdb::SstFileManager* sst_file_manager = nullptr;

if (jlogger_handle != 0) {
auto* sptr_logger =
reinterpret_cast<std::shared_ptr<rocksdb::Logger> *>(jlogger_handle);
sst_file_manager = rocksdb::NewSstFileManager(env, *sptr_logger, "",
jrate_bytes, true, &s, jmax_trash_db_ratio,
jmax_delete_chunk_bytes);
} else {
sst_file_manager = rocksdb::NewSstFileManager(env, nullptr, "",
jrate_bytes, true, &s, jmax_trash_db_ratio,
jmax_delete_chunk_bytes);
}

if (!s.ok()) {
if (sst_file_manager != nullptr) {
delete sst_file_manager;
}
rocksdb::RocksDBExceptionJni::ThrowNew(jnienv, s);
}
auto* sptr_sst_file_manager
= new std::shared_ptr<rocksdb::SstFileManager>(sst_file_manager);

return reinterpret_cast<jlong>(sptr_sst_file_manager);
}

/*
* Class: org_rocksdb_SstFileManager
* Method: setMaxAllowedSpaceUsage
* Signature: (JJ)V
*/
void Java_org_rocksdb_SstFileManager_setMaxAllowedSpaceUsage(
JNIEnv* env, jobject jobj, jlong jhandle, jlong jmax_allowed_space) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
sptr_sst_file_manager->get()->SetMaxAllowedSpaceUsage(jmax_allowed_space);
}

/*
* Class: org_rocksdb_SstFileManager
* Method: setCompactionBufferSize
* Signature: (JJ)V
*/
void Java_org_rocksdb_SstFileManager_setCompactionBufferSize(
JNIEnv* env, jobject jobj, jlong jhandle, jlong jcompaction_buffer_size) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
sptr_sst_file_manager->get()->SetCompactionBufferSize(jcompaction_buffer_size);
}

/*
* Class: org_rocksdb_SstFileManager
* Method: isMaxAllowedSpaceReached
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReached(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
return sptr_sst_file_manager->get()->IsMaxAllowedSpaceReached();
}

/*
* Class: org_rocksdb_SstFileManager
* Method: isMaxAllowedSpaceReachedIncludingCompactions
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReachedIncludingCompactions(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
return sptr_sst_file_manager->get()->IsMaxAllowedSpaceReachedIncludingCompactions();
}

/*
* Class: org_rocksdb_SstFileManager
* Method: getTotalSize
* Signature: (J)J
*/
jlong Java_org_rocksdb_SstFileManager_getTotalSize(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
return sptr_sst_file_manager->get()->GetTotalSize();
}

/*
* Class: org_rocksdb_SstFileManager
* Method: getTrackedFiles
* Signature: (J)Ljava/util/Map;
*/
jobject Java_org_rocksdb_SstFileManager_getTrackedFiles(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
auto tracked_files = sptr_sst_file_manager->get()->GetTrackedFiles();

const jobject jtracked_files = rocksdb::HashMapJni::construct(env,
static_cast<uint32_t>(tracked_files.size()));
if (jtracked_files == nullptr) {
// exception occurred
return nullptr;
}

const rocksdb::HashMapJni::FnMapKV<const std::string, const uint64_t> fn_map_kv =
[env, &tracked_files](const std::pair<const std::string, const uint64_t>& pair) {
const jstring jtracked_file_path = env->NewStringUTF(pair.first.c_str());
if (jtracked_file_path == nullptr) {
// an error occurred
return std::unique_ptr<std::pair<jobject, jobject>>(nullptr);
}
const jobject jtracked_file_size =
rocksdb::LongJni::valueOf(env, pair.second);
if (jtracked_file_size == nullptr) {
// an error occurred
return std::unique_ptr<std::pair<jobject, jobject>>(nullptr);
}
return std::unique_ptr<std::pair<jobject, jobject>>(new std::pair<jobject, jobject>(jtracked_file_path,
jtracked_file_size));
};

if(!rocksdb::HashMapJni::putAll(env, jtracked_files,
tracked_files.begin(), tracked_files.end(), fn_map_kv)) {
// exception occcurred
return nullptr;
}

return jtracked_files;
}

/*
* Class: org_rocksdb_SstFileManager
* Method: getDeleteRateBytesPerSecond
* Signature: (J)J
*/
jlong Java_org_rocksdb_SstFileManager_getDeleteRateBytesPerSecond(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
return sptr_sst_file_manager->get()->GetDeleteRateBytesPerSecond();
}

/*
* Class: org_rocksdb_SstFileManager
* Method: setDeleteRateBytesPerSecond
* Signature: (JJ)V
*/
void Java_org_rocksdb_SstFileManager_setDeleteRateBytesPerSecond(
JNIEnv* env, jobject jobj, jlong jhandle, jlong jdelete_rate) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
sptr_sst_file_manager->get()->SetDeleteRateBytesPerSecond(jdelete_rate);
}

/*
* Class: org_rocksdb_SstFileManager
* Method: getMaxTrashDBRatio
* Signature: (J)D
*/
jdouble Java_org_rocksdb_SstFileManager_getMaxTrashDBRatio(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
return sptr_sst_file_manager->get()->GetMaxTrashDBRatio();
}

/*
* Class: org_rocksdb_SstFileManager
* Method: setMaxTrashDBRatio
* Signature: (JD)V
*/
void Java_org_rocksdb_SstFileManager_setMaxTrashDBRatio(
JNIEnv* env, jobject jobj, jlong jhandle, jdouble jratio) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
sptr_sst_file_manager->get()->SetMaxTrashDBRatio(jratio);
}

/*
* Class: org_rocksdb_SstFileManager
* Method: disposeInternal
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileManager_disposeInternal(
JNIEnv* env, jobject jobj, jlong jhandle) {
auto* sptr_sst_file_manager =
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
delete sptr_sst_file_manager;
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
* {@link AbstractNativeReference} which have an immutable reference to the
* underlying native C++ object
*/
//@ThreadSafe
public abstract class AbstractImmutableNativeReference
extends AbstractNativeReference {

Expand Down
9 changes: 9 additions & 0 deletions java/src/main/java/org/rocksdb/DBOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,13 @@ public DBOptions setRateLimiter(final RateLimiter rateLimiter) {
return this;
}

@Override
public DBOptions setSstFileManager(final SstFileManager sstFileManager) {
assert(isOwningHandle());
setSstFileManager(nativeHandle_, sstFileManager.nativeHandle_);
return this;
}

@Override
public DBOptions setLogger(final Logger logger) {
assert(isOwningHandle());
Expand Down Expand Up @@ -988,6 +995,8 @@ private native void setParanoidChecks(
private native boolean paranoidChecks(long handle);
private native void setRateLimiter(long handle,
long rateLimiterHandle);
private native void setSstFileManager(final long handle,
final long sstFileManagerHandle);
private native void setLogger(long handle,
long loggerHandle);
private native void setInfoLogLevel(long handle, byte logLevel);
Expand Down
19 changes: 19 additions & 0 deletions java/src/main/java/org/rocksdb/DBOptionsInterface.java
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,25 @@ public interface DBOptionsInterface<T extends DBOptionsInterface> {
*/
T setRateLimiter(RateLimiter rateLimiter);

/**
* Use to track SST files and control their file deletion rate.
*
* Features:
* - Throttle the deletion rate of the SST files.
* - Keep track the total size of all SST files.
* - Set a maximum allowed space limit for SST files that when reached
* the DB wont do any further flushes or compactions and will set the
* background error.
* - Can be shared between multiple dbs.
*
* Limitations:
* - Only track and throttle deletes of SST files in
* first db_path (db_name if db_paths is empty).
*
* @param sstFileManager The SST File Manager for the db.
*/
T setSstFileManager(SstFileManager sstFileManager);

/**
* <p>Any internal progress/error information generated by
* the db will be written to the Logger if it is non-nullptr,
Expand Down
9 changes: 9 additions & 0 deletions java/src/main/java/org/rocksdb/Options.java
Original file line number Diff line number Diff line change
Expand Up @@ -1032,6 +1032,13 @@ public Options setRateLimiter(final RateLimiter rateLimiter) {
return this;
}

@Override
public Options setSstFileManager(final SstFileManager sstFileManager) {
assert(isOwningHandle());
setSstFileManager(nativeHandle_, sstFileManager.nativeHandle_);
return this;
}

@Override
public Options setLogger(final Logger logger) {
assert(isOwningHandle());
Expand Down Expand Up @@ -1588,6 +1595,8 @@ private native void setParanoidChecks(
private native boolean paranoidChecks(long handle);
private native void setRateLimiter(long handle,
long rateLimiterHandle);
private native void setSstFileManager(final long handle,
final long sstFileManagerHandle);
private native void setLogger(long handle,
long loggerHandle);
private native void setInfoLogLevel(long handle, byte logLevel);
Expand Down
Loading

0 comments on commit ca87aef

Please sign in to comment.