From 1860066ea7ebedc622e0ae7eaa6ba8b3130441da Mon Sep 17 00:00:00 2001 From: Senthil Nathan Date: Sun, 1 May 2022 22:04:49 -0400 Subject: [PATCH] Changes done for v4.1.7. --- .../distributed/native.function/function.xml | 59 +- .../impl/include/CassandraDBLayer.h | 5 +- .../impl/include/CloudantDBLayer.h | 7 +- .../impl/include/CouchbaseDBLayer.h | 7 +- com.ibm.streamsx.dps/impl/include/DBLayer.h | 18 +- .../impl/include/DistributedProcessStore.h | 472 +++++++--- .../include/DistributedProcessStoreWrappers.h | 64 +- .../impl/include/DpsConstants.h | 21 + .../impl/include/HBaseDBLayer.h | 7 +- .../impl/include/MemcachedDBLayer.h | 7 +- .../impl/include/MongoDBLayer.h | 7 +- .../impl/include/RedisClusterDBLayer.h | 9 +- .../include/RedisClusterPlusPlusDBLayer.h | 9 +- .../impl/include/RedisDBLayer.h | 7 +- .../impl/src/CassandraDBLayer.cpp | 34 +- .../impl/src/CloudantDBLayer.cpp | 35 +- .../impl/src/CouchbaseDBLayer.cpp | 36 +- .../impl/src/HBaseDBLayer.cpp | 35 +- .../impl/src/MemcachedDBLayer.cpp | 35 +- .../impl/src/MongoDBLayer.cpp | 35 +- .../impl/src/RedisClusterDBLayer.cpp | 35 +- .../impl/src/RedisClusterPlusPlusDBLayer.cpp | 745 +++++++++++++-- .../impl/src/RedisDBLayer.cpp | 846 ++++++++++++++---- com.ibm.streamsx.dps/info.xml | 2 +- .../com.acme.test/Main.splmm | 380 ++++++-- .../04_all_dps_apis_at_work_in_spl/info.xml | 4 +- 26 files changed, 2428 insertions(+), 493 deletions(-) diff --git a/com.ibm.streamsx.dps/com.ibm.streamsx/store/distributed/native.function/function.xml b/com.ibm.streamsx.dps/com.ibm.streamsx/store/distributed/native.function/function.xml index 91e8c4c..fc4db34 100644 --- a/com.ibm.streamsx.dps/com.ibm.streamsx/store/distributed/native.function/function.xml +++ b/com.ibm.streamsx.dps/com.ibm.streamsx/store/distributed/native.function/function.xml @@ -435,40 +435,69 @@ it does safety checks and is therefore slower. - This function can be called to get multiple keys present in a given store. + This function can be called to get i.e. read/fetch multiple keys present in a given store. @param store The handle of the store. @param keys User provided mutable list variable. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. - @param keyStartPosition User can indicate a start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. + @param keyStartPosition User can indicate a zero index based start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. @param numberOfKeysNeeded User can indicate the total number of keys to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available keys upto a maximum of 50000 keys from the given key start position will be returned. @param keyExpression User can provide an expression made of the attributes from the key's data type. This expression will be evaluated in determining which matching keys to be returned. Due to very high logic complexity, this feature is not implemented at this time. @param valueExpression User can provide an expression made of the attributes from the value's data type. This expression will be evaluated in determining which matching keys to be returned. Due to very high logic complexity, this feature is not implemented at this time. - @param err Contains the error code. Will be '0' if no error occurs, and a non-zero value otherwise. +@param err User provided mutable uint64 typed variable in which the result of this bulk get keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. <any T1> public stateful void dpsGetKeys(uint64 store, mutable list<T1> keys, int32 keyStartPosition, int32 numberOfKeysNeeded, rstring keyExpression, rstring valueExpression, mutable uint64 err) - This function can be called to get values for a given list of multiple keys present in a given store. + This function can be called to get i.e. read/fetch values for a given list of multiple keys present in a given store. @param store The handle of the store. -@param keys User provided list variable that contains keys for which values need to be fetched. -@param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the key appears in the user provided keys list. Before using the individual values from this list, it is recommended to make sure that a given value fetch worked with no errors. -@param errors User provided mutable list variable in which the individual success or failure value fetch result codes will be returned. This list must be of type uint64. Each list element will be 0 if no error occurs and a non-zero error code otherwise. Such value fetch result codes will be available in this list at the same index where the key appears in the user provided keys list. If a given result code doesn't indicate a successful value fetch, it is better to skip the corresponding element at the same index in the mutable values list. -@return It returns true if value fetch worked for all given keys with no errors. Else, it returns false to indicate that value fetch encountered error for one or more keys. +@param keys User provided list variable that contains keys for which values need to be fetched. It must be made of a given store's key data type. +@param keyExistsOrNot User provided mutable list variable in which true or false status for every user given key will be returned. This is done to indicate to the user about if that key exists or not in the given store. This list must be made of a boolean data type. Status returned in this list can be used in combination with the items returned in the values list which is explained next. +@param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the key appeared in the user provided keys list. If a user given key is not present in the store, there will still be a default value returned in that key's index. It is better to first confirm in the keyExistsOrNot list that will carry true or false status about the existence of all the user provided keys. Depending on the key existence status found in that other list, user can decide to either consider using or ignore a value in a given index of the values list. +@param err User provided mutable uint64 typed variable in which the result of this bulk get values operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. - <any T1, any T2> public stateful boolean dpsGetValues(uint64 store, list<T1> keys, mutable list<T2> values, mutable list<uint64> errors) + <any T1, any T2> public stateful void dpsGetValues(uint64 store, list<T1> keys, mutable list<boolean> keyExistsOrNot, mutable list<T2> values, mutable uint64 err) - This function can be called to get multiple Key/Value (KV) pairs present in a given store. + This function can be called to get i.e. read/fetch multiple Key/Value (KV) pairs present in a given store. @param store The handle of the store. @param keys User provided mutable list variable in which the keys found in the store will be returned. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. -@param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the corresponding key appears in the keys list. Before using the individual values from this list, it is recommended to make sure that a given value fetch worked with no errors. -@param keyStartPosition User can indicate a start position from where the K/V pairs should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. +@param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the corresponding key appears in the keys list. +@param keyStartPosition User can indicate a zero index based start position from where the K/V pairs should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. @param numberOfPairsNeeded User can indicate the total number of K/V pairs to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available K/V pairs upto a maximum of 50000 pairs from the given key start position will be returned. -@param errors User provided mutable list variable in which the individual success or failure value fetch result codes will be returned. This list must be of type uint64. Each list element will be 0 if no error occurs and a non-zero error code otherwise. Such value fetch result codes will be available in this list at the same index where the key appears in the keys list. If a given result code doesn't indicate a successful value fetch, it is better to skip the corresponding element at the same index in the mutable values list. -@return It returns true if value fetch worked for all the keys with no errors. Else, it returns false to indicate that value fetch encountered error for one or more keys. +@param err User provided mutable uint64 typed variable in which the result of this bulk get KV pairs operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. - <any T1, any T2> public stateful boolean dpsGetKVPairs(uint64 store, mutable list<T1> keys, mutable list<T2> values, int32 keyStartPosition, int32 numberOfPairsNeeded, mutable list<uint64> errors) + <any T1, any T2> public stateful void dpsGetKVPairs(uint64 store, mutable list<T1> keys, mutable list<T2> values, int32 keyStartPosition, int32 numberOfPairsNeeded, mutable uint64 err) + + + + This function can be called to put i.e. write/save multiple Key/Value (KV) pairs to a given store. Note: You must make sure that the intended store exists before you use this function. If not, you will end up creating bulk K/V pairs in an unmanageable fashion in the backend data store which will make it less useful. +@param store The handle of the store. +@param keys User provided list variable that contains the keys to be written i.e. saved. This list must be made of a given store's key data type. +@param values User provided list variable that contains the values to be written i.e. saved. This list must be made of a given store's value data type. A KV pair is formed with a key and a value taken from the same index position of the keys and values lists. If the keys and values lists are not of the same size, this function will simply return back without doing any bulk put operation. +@param err User provided mutable uint64 typed variable in which the result of this bulk put operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + + <any T1, any T2> public stateful void dpsPutKVPairs(uint64 store, list<T1> keys, list<T2> values, mutable uint64 err) + + + + This function can be called to check for the existence of a given list of keys in a given store. +@param store The handle of the store. +@param keys User provided list variable that contains the keys to be checked for their existence in the given store. This list must be made of a given store's key data type. +@param results User provided mutable list variable in which the key existence check true or false results will be returned. This list must be made of a boolean data type. +@param err User provided mutable uint64 typed variable in which the result of this bulk has keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + + <any T1> public stateful void dpsHasKeys(uint64 store, list<T1> keys, mutable list<boolean> results, mutable uint64 err) + + + + This function can be called to remove a given list of keys from a given store. +@param store The handle of the store. +@param keys User provided list variable that contains the keys to be removed from the given store. This list must be made of a given store's key data type. +@param totalKeysRemoved User provided mutable int32 variable in which the total number of keys removed from the store will be returned. +@param err User provided mutable uint64 typed variable in which the result of this bulk remove keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + + <any T1> public stateful void dpsRemoveKeys(uint64 store, list<T1> keys, mutable int32 totalKeysRemoved, mutable uint64 err) diff --git a/com.ibm.streamsx.dps/impl/include/CassandraDBLayer.h b/com.ibm.streamsx.dps/impl/include/CassandraDBLayer.h index 6d12a97..c9cdf7c 100644 --- a/com.ibm.streamsx.dps/impl/include/CassandraDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/CassandraDBLayer.h @@ -159,7 +159,10 @@ namespace distributed bool removeLock(uint64_t lock, PersistenceError & lkError); uint32_t getPidForLock(std::string const & name, PersistenceError & lkError); void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); }; } } } } } diff --git a/com.ibm.streamsx.dps/impl/include/CloudantDBLayer.h b/com.ibm.streamsx.dps/impl/include/CloudantDBLayer.h index e66b1d0..609e835 100644 --- a/com.ibm.streamsx.dps/impl/include/CloudantDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/CloudantDBLayer.h @@ -197,8 +197,11 @@ namespace distributed bool acquireLock(uint64_t lock, double leaseTime, double maxWaitTimeToAcquireLock, PersistenceError & lkError); bool removeLock(uint64_t lock, PersistenceError & lkError); uint32_t getPidForLock(std::string const & name, PersistenceError & lkError); - void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); }; } } } } } diff --git a/com.ibm.streamsx.dps/impl/include/CouchbaseDBLayer.h b/com.ibm.streamsx.dps/impl/include/CouchbaseDBLayer.h index 24e22e5..d5bc2fe 100644 --- a/com.ibm.streamsx.dps/impl/include/CouchbaseDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/CouchbaseDBLayer.h @@ -203,8 +203,11 @@ namespace distributed bool acquireLock(uint64_t lock, double leaseTime, double maxWaitTimeToAcquireLock, PersistenceError & lkError); bool removeLock(uint64_t lock, PersistenceError & lkError); uint32_t getPidForLock(std::string const & name, PersistenceError & lkError); - void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); }; } } } } } diff --git a/com.ibm.streamsx.dps/impl/include/DBLayer.h b/com.ibm.streamsx.dps/impl/include/DBLayer.h index ff51126..f5442ed 100644 --- a/com.ibm.streamsx.dps/impl/include/DBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/DBLayer.h @@ -289,9 +289,21 @@ namespace store { /// @return a list containing multiple keys of a given data type. virtual void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError) = 0; - /// Get value for a given key present in a given store. - /// @return a value for a given key of a given data type. - virtual void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) = 0; + /// Get values for a given list of keys present in a given store. + /// @return a list containing multiple values for a given list of keys of a given data type. + virtual void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) = 0; + + /// Put (save) K/V pairs in a given store. + /// @return 0 if all put operations succeeded. Else, non-zero if one or more or all put failed. + virtual void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) = 0; + + /// Check for the existence of a given list of keys in a given store. + /// @return a list containing the key existence check true or false results. + virtual void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) = 0; + + /// Remove a given list of keys from a given store. + /// @return an integer value indicating the total number of keys removed. + virtual void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) = 0; /// A store iterator class Iterator diff --git a/com.ibm.streamsx.dps/impl/include/DistributedProcessStore.h b/com.ibm.streamsx.dps/impl/include/DistributedProcessStore.h index a59f523..250aab5 100644 --- a/com.ibm.streamsx.dps/impl/include/DistributedProcessStore.h +++ b/com.ibm.streamsx.dps/impl/include/DistributedProcessStore.h @@ -10,6 +10,7 @@ #include "PersistenceError.h" #include "DBLayer.h" +#include "DpsConstants.h" #include #include @@ -221,35 +222,62 @@ namespace distributed /// Get multiple keys present in a given store. /// @param store The handle of the store. /// @param keys User provided mutable list variable. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. - /// @param keyStartPosition User can indicate a start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. + /// @param keyStartPosition User can indicate a zero index based start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. /// @param numberOfKeysNeeded User can indicate the total number of keys to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available keys upto a maximum of 50000 keys from the given key start position will be returned. /// @param keyExpression User can provide an expression made of the attributes from the key's data type. This expression will be evaluated in determining which matching keys to be returned. [This feature is not implemented at this time.] /// @param valueExpression User can provide an expression made of the attributes from the value's data type. This expression will be evaluated in determining which matching keys to be returned. [This feature is not implemented at this time.] - /// @param err Contains the error code. Will be '0' if no error occurs, and a non-zero value otherwise. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template void getKeys(SPL::uint64 store, SPL::list & keys, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfKeysNeeded, SPL::rstring const & keyExpression, SPL::rstring const & valueExpression, SPL::uint64 & err); - /// This function can be called to get values for a given list of multiple keys present in a given store. + /// This function can be called to get i.e. read/fetch values for a given list of multiple keys present in a given store. /// @param store The handle of the store. - /// @param keys User provided list variable that contains keys for which values need to be fetched. - /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the key appeared in the user provided keys list. Before using the individual values from this list, it is recommended to make sure that a given value fetch worked with no errors. - /// @param errors User provided mutable list variable in which the individual success or failure value fetch result codes will be returned. This list must be of type uint64. Each list element will be 0 if no error occurs and a non-zero error code otherwise. Such value fetch result codes will be available in this list at the same index where the key appears in the user provided keys list. If a given result code doesn't indicate a successful value fetch, it is better to skip the corresponding element at the same index in the mutable values list. + /// @param keys User provided list variable that contains keys for which values need to be fetched. It must be made of a given store's key data type. + /// @param keyExistsOrNot User provided mutable list variable in which true or false status for every user given key will be returned. This is done to indicate to the user about if that key exists or not in the given store. This list must be made of a boolean data type. Status returned in this list can be used in combination with the items returned in the values list which is explained next. + /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the key appeared in the user provided keys list. If a user given key is not present in the store, there will still be a default value returned in that key's index. It is better to first confirm in the keyExistsOrNot list that will carry true or false status about the existence of all the user provided keys. Depending on the key existence status found in that other list, user can decide to either consider using or ignore a value in a given index of the values list. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get values operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template - bool getValues(SPL::uint64 store, SPL::list const & keys, SPL::list & values, SPL::list & errors); + void getValues(SPL::uint64 store, SPL::list const & keys, SPL::list & keyExistsOrNot, SPL::list & values, SPL::uint64 & err); /// This function can be called to get multiple Key/Value (KV) pairs present in a given store. /// @param store The handle of the store. /// @param keys User provided mutable list variable in which the keys found in the store will be returned. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. - /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the corresponding key appears in the keys list. Before using the individual values from this list, it is recommended to make sure that a given value fetch worked with no errors. - /// @param keyStartPosition User can indicate a start position from where the K/V pairs should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. + /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the corresponding key appears in the keys list. + /// @param keyStartPosition User can indicate a zero index based start position from where the K/V pairs should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. /// @param numberOfPairsNeeded User can indicate the total number of K/V pairs to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available K/V pairs upto a maximum of 50000 pairs from the given key start position will be returned. - /// @param errors User provided mutable list variable in which the individual success or failure value fetch result codes will be returned. This list must be of type uint64. Each list element will be 0 if no error occurs and a non-zero error code otherwise. Such value fetch result codes will be available in this list at the same index where the key appears in the keys list. If a given result code doesn't indicate a successful value fetch, it is better to skip the corresponding element at the same index in the mutable values list. - /// @return It returns true if value fetch worked for all the keys with no errors. Else, it returns false to indicate that value fetch encountered error for one or more keys. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get KV pairs operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template - bool getKVPairs(SPL::uint64 store, SPL::list & keys, SPL::list & values, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfPairsNeeded, SPL::list & errors); + void getKVPairs(SPL::uint64 store, SPL::list & keys, SPL::list & values, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfPairsNeeded, SPL::uint64 & err); + + /// This function can be called to put i.e. write/save multiple Key/Value (KV) pairs to a given store. + /// @param store The handle of the store. + /// @param keys User provided list variable that contains the keys to be written i.e. saved. This list must be made of a given store's key data type. + /// @param values User provided list variable that contains the values to be written i.e. saved. This list must be made of a given store's value data type. A KV pair is formed with a key and a value taken from the same index position of the keys and values lists. If the keys and values lists are not of the same size, this function will simply return back without doing any bulk put operation. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk put operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + /// + template + void putKVPairs(SPL::uint64 store, SPL::list const & keys, SPL::list const & values, SPL::uint64 & err); + + /// This function can be called to check for the existence of a given list of keys in a given store. + /// @param store The handle of the store. + /// @param keys User provided list variable that contains the keys to be checked for their existence in the given store. This list must be made of a given store's key data type. + /// @param results User provided mutable list variable in which the key existence check true or false results will be returned. This list must be made of a boolean data type. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk has keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + /// + template + void hasKeys(SPL::uint64 store, SPL::list const & keys, SPL::list & results, SPL::uint64 & err); + + /// This function can be called to remove a given list of keys from a given store. + /// @param store The handle of the store. + /// @param keys User provided list variable that contains the keys to be removed from the given store. This list must be made of a given store's key data type. + /// @param totalKeysRemoved User provided mutable int32 variable in which the total number of keys removed from the store will be returned. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk remove keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + /// + template + void removeKeys(SPL::uint64 store, SPL::list const & keys, SPL::int32 & totalKeysRemoved, SPL::uint64 & err); /// Serialize the items from the serialized store /// @param store store handle @@ -894,151 +922,347 @@ namespace distributed /// Get multiple keys present in a given store. /// @param store The handle of the store. /// @param keys User provided mutable list variable. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. - /// @param keyStartPosition User can indicate a start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. + /// @param keyStartPosition User can indicate a zero index based start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. /// @param numberOfKeysNeeded User can indicate the total number of keys to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available keys upto a maximum of 50000 keys from the given key start position will be returned. /// @param keyExpression User can provide an expression made of the attributes from the key's data type. This expression will be evaluated in determining which matching keys to be returned. [This feature is not implemented at this time.] /// @param valueExpression User can provide an expression made of the attributes from the value's data type. This expression will be evaluated in determining which matching keys to be returned. [This feature is not implemented at this time.] - /// @param err Contains the error code. Will be '0' if no error occurs, and a non-zero value otherwise. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template void DistributedProcessStore::getKeys(SPL::uint64 store, SPL::list & keys, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfKeysNeeded, SPL::rstring const & keyExpression, SPL::rstring const & valueExpression, SPL::uint64 & err) { - dbError_->reset(); - std::vector keysBuffer; - std::vector keysSize; - // Clear the user provided list now. - keys.clear(); - // Call the underlying store implementation function to get multiple keys in a given store. - db_->getKeys(store, keysBuffer, keysSize, keyStartPosition, numberOfKeysNeeded, *dbError_); - err = dbError_->getErrorCode(); - - if(err != 0) { - // We got an error. Return now without populating anything in the - // user provided list (vector). We will free any memory allocated - // for the partial set of keys returned if any. - for (unsigned int i = 0; i < keysBuffer.size(); i++) { - if(keysSize.at(i) > 0) { - // We must free the buffer allocated by the underlying implementation that we called above. - free(keysBuffer.at(i)); - } - } // End of for loop. + dbError_->reset(); + std::vector keysBuffer; + std::vector keysSize; + // Clear the user provided list now. + keys.clear(); + // Call the underlying store implementation function to get multiple keys in a given store. + db_->getKeys(store, keysBuffer, keysSize, keyStartPosition, numberOfKeysNeeded, *dbError_); + err = dbError_->getErrorCode(); + + if(err != 0) { + // We got an error. Return now without populating anything in the + // user provided list (vector). We will free any memory allocated + // for the partial set of keys returned if any. + for (unsigned int i = 0; i < keysBuffer.size(); i++) { + if(keysSize.at(i) > 0) { + // We must free the buffer allocated by the underlying implementation that we called above. + free(keysBuffer.at(i)); + } + } // End of for loop. - return; - } - - // We got multiple keys. - // Let us convert it to the proper key type and store them in - // the user provided list (vector). - // - // Populate the user provided list (vector) with the store keys. - for (unsigned int i = 0; i < keysBuffer.size(); i++) { - SPL::NativeByteBuffer nbf_key(keysBuffer.at(i), keysSize.at(i)); + return; + } + + // We got multiple keys. + // Let us convert it to the proper key type and store them in + // the user provided list (vector). + // + // Populate the user provided list (vector) with the store keys. + for (unsigned int i = 0; i < keysBuffer.size(); i++) { + SPL::NativeByteBuffer nbf_key(keysBuffer.at(i), keysSize.at(i)); T1 tempKey; // Deserialize the obtained keys from their NBF format to their native SPL type. - nbf_key >> tempKey; + nbf_key >> tempKey; keys.push_back(tempKey); if(keysSize.at(i) > 0) { // We must free the buffer allocated by the underlying implementation that we called above. free(keysBuffer.at(i)); } - } // End of for loop. + } // End of for loop. } // End of getKeys method. - /// This function can be called to get values for a given list of multiple keys present in a given store. + /// This function can be called to get i.e. read/fetch values for a given list of multiple keys present in a given store. /// @param store The handle of the store. - /// @param keys User provided list variable that contains keys for which values need to be fetched. - /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the key appeared in the user provided keys list. Before using the individual values from this list, it is recommended to make sure that a given value fetch worked with no errors. - /// @param errors User provided mutable list variable in which the individual success or failure value fetch result codes will be returned. This list must be of type uint64. Each list element will be 0 if no error occurs and a non-zero error code otherwise. Such value fetch result codes will be available in this list at the same index where the key appears in the user provided keys list. If a given result code doesn't indicate a successful value fetch, it is better to skip the corresponding element at the same index in the mutable values list. + /// @param keys User provided list variable that contains keys for which values need to be fetched. It must be made of a given store's key data type. + /// @param keyExistsOrNot User provided mutable list variable in which true or false status for every user given key will be returned. This is done to indicate to the user about if that key exists or not in the given store. This list must be made of a boolean data type. Status returned in this list can be used in combination with the items returned in the values list which is explained next. + /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the key appeared in the user provided keys list. If a user given key is not present in the store, there will still be a default value returned in that key's index. It is better to first confirm in the keyExistsOrNot list that will carry true or false status about the existence of all the user provided keys. Depending on the key existence status found in that other list, user can decide to either consider using or ignore a value in a given index of the values list. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get values operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template - bool DistributedProcessStore::getValues(SPL::uint64 store, SPL::list const & keys, SPL::list & values, SPL::list & errors) { - dbError_->reset(); - bool resultStatus = true; - - // Clear the user provided mutable lists now. - values.clear(); - errors.clear(); - - // Create a string form of the store id. - std::ostringstream storeId; - storeId << store; - std::string storeIdString = storeId.str(); - - // Populate the key buffer with the serialized key. - // i.e. Serialize it from the native SPL type to NBF format. - // Then, call the method in the underlying DB implementation layer for every key that we have. - for(int i=0; igetValue(storeIdString, keyData, keySize, valueData, valueSize, error); - - // Collect the results in the user provided list. - T2 tempValue; - - if(error == 0) { - SPL::NativeByteBuffer value_nbf(valueData, valueSize); - value_nbf >> tempValue; - } else { - // At least one error seen while fetching the value for a key. - resultStatus = false; - } - - values.push_back(tempValue); - errors.push_back(error); + void DistributedProcessStore::getValues(SPL::uint64 store, SPL::list const & keys, SPL::list & keyExistsOrNot, SPL::list & values, SPL::uint64 & err) { + dbError_->reset(); + + // Clear the user provided mutable lists now. + keyExistsOrNot.clear(); + values.clear(); + + if(keys.size() == 0) { + // Caller sent us no keys. + // Set an error code so that the caller knows that there is no real get values done by us here. + dbError_->set(std::string("Inside the top level getValues #1, this method was called with an empty keys list."), DPS_GET_VALUES_EMPTY_KEYS_LIST_ERROR); + err = dbError_->getErrorCode(); + return; + } + + // We will create the NBF based keys and values, store them in different lists and + // send it to the underlying DB layer for it to do bulk writes of the K/V pairs. + std::vector keyData; + std::vector keySize; + std::vector valueData; + std::vector valueSize; + + std::vector key_nbf; + + // We have to have NBF objects in scope at the time of calling + // the underlying DB layer functions. We will keep them in a vector. + for(int i=0; igetValues(store, keyData, keySize, valueData, valueSize, *dbError_); + err = dbError_->getErrorCode(); + + if(err != 0) { + // Error occurred. + // We will release the allocated memory buffers if any by the underlying DB layer before we return from here. + for(int i=0; i 0) { + free(valueData.at(i)); + } + } - // We must free the memory allocated in the DB layer. - if(valueSize > 0) { - free(valueData); - } - } // End of for loop. + return; + } + + // We have to now turn the value data read from the backend store into proper + // SPL type so that it is consumable by the caller. + for(int i=0; i> tempValue; + // We must free the memory allocated in the DB layer. + free(valueData.at(i)); + + // Indicate about the existence of the key in the user provided list. + keyExistsOrNot.push_back(true); + } else { + // Indicate about the non-existence of the key in the user provided list. + keyExistsOrNot.push_back(false); + } - return(resultStatus); + // To the user provided values list, add either a fetched good value result for a key that exists in the + // store or a default value for a key that doesn't exist in the store. + values.push_back(tempValue); + } // End of for loop. } // End of getValues method. /// This function can be called to get multiple Key/Value (KV) pairs present in a given store. /// @param store The handle of the store. /// @param keys User provided mutable list variable in which the keys found in the store will be returned. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. - /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the corresponding key appears in the keys list. Before using the individual values from this list, it is recommended to make sure that a given value fetch worked with no errors. - /// @param keyStartPosition User can indicate a start position from where the K/V pairs should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. + /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the corresponding key appears in the keys list. + /// @param keyStartPosition User can indicate a zero index based start position from where the K/V pairs should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. /// @param numberOfPairsNeeded User can indicate the total number of K/V pairs to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available K/V pairs upto a maximum of 50000 pairs from the given key start position will be returned. - /// @param errors User provided mutable list variable in which the individual success or failure value fetch result codes will be returned. This list must be of type uint64. Each list element will be 0 if no error occurs and a non-zero error code otherwise. Such value fetch result codes will be available in this list at the same index where the key appears in the keys list. If a given result code doesn't indicate a successful value fetch, it is better to skip the corresponding element at the same index in the mutable values list. - /// @return It returns true if value fetch worked for all the keys with no errors. Else, it returns false to indicate that value fetch encountered error for one or more keys. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get KV pairs operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template - bool DistributedProcessStore::getKVPairs(SPL::uint64 store, SPL::list & keys, SPL::list & values, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfPairsNeeded, SPL::list & errors) { - // The logic in this method is going to be a combination of the two previous methods shown above. - // We have to first get the keys and then the values for those keys. - dbError_->reset(); - bool resultStatus = true; - uint64_t error = 0; - std::string keyExpression = ""; - std::string valueExpression = ""; - - // Clear the user provided mutable lists now. - keys.clear(); - values.clear(); - errors.clear(); - - // Let us first get the keys available in the user specified range. - getKeys(store, keys, keyStartPosition, numberOfPairsNeeded, keyExpression, valueExpression, error); - - // If we encountered an error in getting the keys, we can return back from here. - if(error != 0) { - resultStatus = false; - return(resultStatus); - } + void DistributedProcessStore::getKVPairs(SPL::uint64 store, SPL::list & keys, SPL::list & values, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfPairsNeeded, SPL::uint64 & err) { + // The logic in this method is going to be a combination of the two previous methods shown above. + // We have to first get the keys and then the values for those keys. + dbError_->reset(); + std::string keyExpression = ""; + std::string valueExpression = ""; + + // Clear the user provided mutable lists now. + keys.clear(); + values.clear(); + + // Let us first get the keys available in the user specified range. + getKeys(store, keys, keyStartPosition, numberOfPairsNeeded, keyExpression, valueExpression, err); + + // If we encountered an error in getting the keys, we can return back from here. + if(err != 0) { + return; + } + + SPL::list keyExistsOrNot; - // We got the keys. Let us now get the values stored for those keys. - resultStatus = getValues(store, keys, values, errors); - return(resultStatus); + // We got the keys. Let us now get the values stored for those keys. + getValues(store, keys, keyExistsOrNot, values, err); } // End of getKVPairs method. + /// This function can be called to put i.e. write/save multiple Key/Value (KV) pairs to a given store. + /// @param store The handle of the store. + /// @param keys User provided list variable that contains the keys to be written i.e. saved. This list must be made of a given store's key data type. + /// @param values User provided list variable that contains the values to be written i.e. saved. This list must be made of a given store's value data type. A KV pair is formed with a key and a value taken from the same index position of the keys and values lists. If the keys and values lists are not of the same size, this function will simply return back without doing any bulk put operation. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk put operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + /// + template + void DistributedProcessStore::putKVPairs(SPL::uint64 store, SPL::list const & keys, SPL::list const & values, SPL::uint64 & err) { + dbError_->reset(); + + // Check if the two user provided keys and values lists are of the same size. + if(keys.size() != values.size()) { + // They are not of the same size. We will ignore the caller's intent and + // return back from here without doing any useful work. + // Set an error code so that the caller knows that there is no real put KV pairs done by us here. + dbError_->set(std::string("Inside the top level putKVPairs #1, this method was called with keys and values lists that are made of different sizes."), DPS_PUT_KV_PAIRS_KEYS_VALUES_LISTS_NOT_OF_SAME_SIZE_ERROR); + err = dbError_->getErrorCode(); + return; + } + + // We will create the NBF based keys and values, store them in different lists and + // send it to the underlying DB layer for it to do bulk writes of the K/V pairs. + std::vector keyData; + std::vector keySize; + std::vector valueData; + std::vector valueSize; + + std::vector key_nbf; + std::vector value_nbf; + + // We have to have NBF objects in scope at the time of calling + // the underlying DB layer functions. We will keep them in a vector. + for(int i=0; iputKVPairs(store, keyData, keySize, valueData, valueSize, *dbError_); + err = dbError_->getErrorCode(); + } // End of putKVPairs method. + + /// This function can be called to check for the existence of a given list of keys in a given store. + /// @param store The handle of the store. + /// @param keys User provided list variable that contains the keys to be checked for their existence in the given store. This list must be made of a given store's key data type. + /// @param results User provided mutable list variable in which the key existence check true or false results will be returned. This list must be made of a boolean data type. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk has keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + /// + template + void DistributedProcessStore::hasKeys(SPL::uint64 store, SPL::list const & keys, SPL::list & results, SPL::uint64 & err) { + dbError_->reset(); + + // Clear the user provided mutable list now. + results.clear(); + + if(keys.size() == 0) { + // Caller sent us no keys. + // Set an error code so that the caller knows that there are no existence results sent from us here. + dbError_->set(std::string("Inside the top level hasKeys #1, this method was called with an empty keys list."), DPS_HAS_KEYS_EMPTY_KEYS_LIST_ERROR); + err = dbError_->getErrorCode(); + return; + } + + // We will create the NBF based keys, store them in a different list and + // send it to the underlying DB layer for it to do bulk existence check of those keys. + std::vector keyData; + std::vector keySize; + + std::vector key_nbf; + + // We have to have NBF objects in scope at the time of calling + // the underlying DB layer functions. We will keep them in a vector. + for(int i=0; i to the + // underlying DB implementation that expects std::vector. + // So, I'm having this local variable to collect the results and then + // transfer it to the user provided list right after the call returns back. + // There must be an easier way to avoid this use of two different array + // types to do one task. It is a waste of extra logic and memory copy. + // At a later time, try to simplify it. + std::vector myResults; + + // Call the underlying DB implementation layer only once to do bulk existence check for a given list of keys. + db_->hasKeys(store, keyData, keySize, myResults, *dbError_); + err = dbError_->getErrorCode(); + + // Populate the user provided SPL::list + for(int i=0; i + void DistributedProcessStore::removeKeys(SPL::uint64 store, SPL::list const & keys, SPL::int32 & totalKeysRemoved, SPL::uint64 & err) { + dbError_->reset(); + + // Reset the result variable. + totalKeysRemoved = 0; + + if(keys.size() == 0) { + // Caller sent us no keys. + // Set an error code so that the caller knows that there are no key removals done here. + dbError_->set(std::string("Inside the top level removeKeys #1, this method was called with an empty keys list."), DPS_REMOVE_KEYS_EMPTY_KEYS_LIST_ERROR); + err = dbError_->getErrorCode(); + return; + } + + // We will create the NBF based keys, store them in a different list and + // send it to the underlying DB layer for it to do bulk existence check of those keys. + std::vector keyData; + std::vector keySize; + + std::vector key_nbf; + + // We have to have NBF objects in scope at the time of calling + // the underlying DB layer functions. We will keep them in a vector. + for(int i=0; iremoveKeys(store, keyData, keySize, totalKeysRemoved, *dbError_); + err = dbError_->getErrorCode(); + } // End of removeKeys method. + template void DistributedProcessStore::serialize(SPL::uint64 store, SPL::blob & data, SPL::uint64 & err) { diff --git a/com.ibm.streamsx.dps/impl/include/DistributedProcessStoreWrappers.h b/com.ibm.streamsx.dps/impl/include/DistributedProcessStoreWrappers.h index 49a7030..dfb8684 100644 --- a/com.ibm.streamsx.dps/impl/include/DistributedProcessStoreWrappers.h +++ b/com.ibm.streamsx.dps/impl/include/DistributedProcessStoreWrappers.h @@ -303,11 +303,11 @@ namespace distributed /// Get multiple keys present in a given store. /// @param store The handle of the store. /// @param keys User provided mutable list variable. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. - /// @param keyStartPosition User can indicate a start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. + /// @param keyStartPosition User can indicate a zero index based start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. /// @param numberOfKeysNeeded User can indicate the total number of keys to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available keys upto a maximum of 50000 keys from the given key start position will be returned. /// @param keyExpression User can provide an expression made of the attributes from the key's data type. This expression will be evaluated in determining which matching keys to be returned. [This feature is not implemented at this time.] /// @param valueExpression User can provide an expression made of the attributes from the value's data type. This expression will be evaluated in determining which matching keys to be returned. [This feature is not implemented at this time.] - /// @param err Contains the error code. Will be '0' if no error occurs, and a non-zero value otherwise. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template void dpsGetKeys(SPL::uint64 store, SPL::list & keys, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfKeysNeeded, SPL::rstring const & keyExpression, SPL::rstring const & valueExpression, SPL::uint64 & err) @@ -315,33 +315,69 @@ namespace distributed return DistributedProcessStore::getGlobalStore().getKeys(store, keys, keyStartPosition, numberOfKeysNeeded, keyExpression, valueExpression, err); } - /// This function can be called to get values for a given list of multiple keys present in a given store. + /// This function can be called to get i.e. read/fetch values for a given list of multiple keys present in a given store. /// @param store The handle of the store. - /// @param keys User provided list variable that contains keys for which values need to be fetched. - /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the key appeared in the user provided keys list. Before using the individual values from this list, it is recommended to make sure that a given value fetch worked with no errors. - /// @param errors User provided mutable list variable in which the individual success or failure value fetch result codes will be returned. This list must be of type uint64. Each list element will be 0 if no error occurs and a non-zero error code otherwise. Such value fetch result codes will be available in this list at the same index where the key appears in the user provided keys list. If a given result code doesn't indicate a successful value fetch, it is better to skip the corresponding element at the same index in the mutable values list. + /// @param keys User provided list variable that contains keys for which values need to be fetched. It must be made of a given store's key data type. + /// @param keyExistsOrNot User provided mutable list variable in which true or false status for every user given key will be returned. This is done to indicate to the user about if that key exists or not in the given store. This list must be made of a boolean data type. Status returned in this list can be used in combination with the items returned in the values list which is explained next. + /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the key appeared in the user provided keys list. If a user given key is not present in the store, there will still be a default value returned in that key's index. It is better to first confirm in the keyExistsOrNot list that will carry true or false status about the existence of all the user provided keys. Depending on the key existence status found in that other list, user can decide to either consider using or ignore a value in a given index of the values list. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get values operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template - bool dpsGetValues(SPL::uint64 store, SPL::list const & keys, SPL::list & values, SPL::list & errors) + void dpsGetValues(SPL::uint64 store, SPL::list const & keys, SPL::list & keyExistsOrNot, SPL::list & values, SPL::uint64 & err) { - return DistributedProcessStore::getGlobalStore().getValues(store, keys, values, errors); + return DistributedProcessStore::getGlobalStore().getValues(store, keys, keyExistsOrNot, values, err); } /// This function can be called to get multiple Key/Value (KV) pairs present in a given store. /// @param store The handle of the store. /// @param keys User provided mutable list variable in which the keys found in the store will be returned. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. - /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the corresponding key appears in the keys list. Before using the individual values from this list, it is recommended to make sure that a given value fetch worked with no errors. - /// @param keyStartPosition User can indicate a start position from where the K/V pairs should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. + /// @param values User provided mutabe list variable in which the fetched values will be returned. This list must be suitable for storing multiple values obtained from a given store and it must be made of a given store's value data type. Fetched values will be available in this list at the same index where the corresponding key appears in the keys list. + /// @param keyStartPosition User can indicate a zero index based start position from where the K/V pairs should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. /// @param numberOfPairsNeeded User can indicate the total number of K/V pairs to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available K/V pairs upto a maximum of 50000 pairs from the given key start position will be returned. - /// @param errors User provided mutable list variable in which the individual success or failure value fetch result codes will be returned. This list must be of type uint64. Each list element will be 0 if no error occurs and a non-zero error code otherwise. Such value fetch result codes will be available in this list at the same index where the key appears in the keys list. If a given result code doesn't indicate a successful value fetch, it is better to skip the corresponding element at the same index in the mutable values list. - /// @return It returns true if value fetch worked for all the keys with no errors. Else, it returns false to indicate that value fetch encountered error for one or more keys. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get KV pairs operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// template - bool dpsGetKVPairs(SPL::uint64 store, SPL::list & keys, SPL::list & values, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfPairsNeeded, SPL::list & errors) + void dpsGetKVPairs(SPL::uint64 store, SPL::list & keys, SPL::list & values, SPL::int32 const & keyStartPosition, SPL::int32 const & numberOfPairsNeeded, SPL::uint64 & err) { - return DistributedProcessStore::getGlobalStore().getKVPairs(store, keys, values, keyStartPosition, numberOfPairsNeeded, errors); + return DistributedProcessStore::getGlobalStore().getKVPairs(store, keys, values, keyStartPosition, numberOfPairsNeeded, err); } + /// This function can be called to put i.e. write/save multiple Key/Value (KV) pairs to a given store. + /// @param store The handle of the store. + /// @param keys User provided list variable that contains the keys to be written i.e. saved. This list must be made of a given store's key data type. + /// @param values User provided list variable that contains the values to be written i.e. saved. This list must be made of a given store's value data type. A KV pair is formed with a key and a value taken from the same index position of the keys and values lists. If the keys and values lists are not of the same size, this function will simply return back without doing any bulk put operation. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk put operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + /// + template + void dpsPutKVPairs(SPL::uint64 store, SPL::list const & keys, SPL::list const & values, SPL::uint64 & err) + { + return DistributedProcessStore::getGlobalStore().putKVPairs(store, keys, values, err); + } + + /// This function can be called to check for the existence of a given list of keys in a given store. + /// @param store The handle of the store. + /// @param keys User provided list variable that contains the keys to be checked for their existence in the given store. This list must be made of a given store's key data type. + /// @param results User provided mutable list variable in which the key existence check true or false results will be returned. This list must be made of a boolean data type. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk has keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + /// + template + void dpsHasKeys(SPL::uint64 store, SPL::list const & keys, SPL::list & results, SPL::uint64 & err) + { + return DistributedProcessStore::getGlobalStore().hasKeys(store, keys, results, err); + } + + /// This function can be called to remove a given list of keys from a given store. + /// @param store The handle of the store. + /// @param keys User provided list variable that contains the keys to be removed from the given store. This list must be made of a given store's key data type. + /// @param totalKeysRemoved User provided mutable int32 variable in which the total number of keys removed from the store will be returned. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk remove keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. + /// + template + void dpsRemoveKeys(SPL::uint64 store, SPL::list const & keys, SPL::int32 & totalKeysRemoved, SPL::uint64 & err) + { + return DistributedProcessStore::getGlobalStore().removeKeys(store, keys, totalKeysRemoved, err); + } + /// Serialize the items from the serialized store /// @param store store handle /// @param data blob to serialize into diff --git a/com.ibm.streamsx.dps/impl/include/DpsConstants.h b/com.ibm.streamsx.dps/impl/include/DpsConstants.h index 50209bb..16f4ad8 100644 --- a/com.ibm.streamsx.dps/impl/include/DpsConstants.h +++ b/com.ibm.streamsx.dps/impl/include/DpsConstants.h @@ -81,6 +81,9 @@ interface with many different back-end in-memory stores. #define REDIS_ZADD_CMD "zadd " #define REDIS_ZREM_CMD "zrem " #define REDIS_ZRANGE_CMD "zrange " +#define REDIS_HMSET_CMD "hmset " +#define REDIS_HMGET_CMD "hmget " +#define REDIS_ZMSCORE_CMD "zmscore " #define CASSANDRA_DPS_KEYSPACE "com_ibm_streamsx_dps" #define CASSANDRA_DPS_MAIN_TABLE "t1" #define HBASE_DPS_MAIN_TABLE "dps_t1" @@ -221,6 +224,24 @@ interface with many different back-end in-memory stores. #define DPS_REDIS_REPLY_NULL_ERROR 161 #define DPS_REDIS_REPLY_NIL_ERROR 162 #define DPS_EMPTY_DATA_ITEM_VALUE_FOUND_ERROR 163 +#define DPS_BULK_PUT_HSET_ERROR 164 +#define DPS_BULK_PUT_ZADD_ERROR 165 +#define DPS_BULK_GET_HMGET_ERROR 166 +#define DPS_BULK_GET_HMGET_NO_REPLY_ARRAY_ERROR 167 +#define DPS_BULK_GET_HMGET_MALLOC_ERROR 168 +#define DPS_BULK_GET_HMGET_EMPTY_VALUE_ERROR 169 +#define DPS_BULK_GET_ZMSCORE_ERROR 170 +#define DPS_BULK_GET_ZMSCORE_NO_REPLY_ARRAY_ERROR 171 +#define DPS_BULK_GET_ZMSCORE_EMPTY_VALUE_ERROR 172 +#define DPS_BULK_REMOVE_HDEL_ERROR 173 +#define DPS_BULK_REMOVE_ZREM_ERROR 174 +#define DPS_BULK_REMOVE_CNT_MISMATCH_ERROR 175 +#define DPS_BULK_REMOVE_HDEL_NO_INTEGER_REPLY_ERROR 176 +#define DPS_BULK_REMOVE_ZREM_NO_INTEGER_REPLY_ERROR 177 +#define DPS_GET_VALUES_EMPTY_KEYS_LIST_ERROR 178 +#define DPS_PUT_KV_PAIRS_KEYS_VALUES_LISTS_NOT_OF_SAME_SIZE_ERROR 179 +#define DPS_HAS_KEYS_EMPTY_KEYS_LIST_ERROR 180 +#define DPS_REMOVE_KEYS_EMPTY_KEYS_LIST_ERROR 181 #define DL_CONNECTION_ERROR 501 #define DL_GET_LOCK_ID_ERROR 502 diff --git a/com.ibm.streamsx.dps/impl/include/HBaseDBLayer.h b/com.ibm.streamsx.dps/impl/include/HBaseDBLayer.h index 4b0e966..0564df9 100644 --- a/com.ibm.streamsx.dps/impl/include/HBaseDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/HBaseDBLayer.h @@ -212,8 +212,11 @@ namespace distributed bool acquireLock(uint64_t lock, double leaseTime, double maxWaitTimeToAcquireLock, PersistenceError & lkError); bool removeLock(uint64_t lock, PersistenceError & lkError); uint32_t getPidForLock(std::string const & name, PersistenceError & lkError); - void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); }; } } } } } diff --git a/com.ibm.streamsx.dps/impl/include/MemcachedDBLayer.h b/com.ibm.streamsx.dps/impl/include/MemcachedDBLayer.h index 25daf29..0543ff0 100644 --- a/com.ibm.streamsx.dps/impl/include/MemcachedDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/MemcachedDBLayer.h @@ -178,8 +178,11 @@ namespace distributed bool acquireLock(uint64_t lock, double leaseTime, double maxWaitTimeToAcquireLock, PersistenceError & lkError); bool removeLock(uint64_t lock, PersistenceError & lkError); uint32_t getPidForLock(std::string const & name, PersistenceError & lkError); - void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); }; } } } } } diff --git a/com.ibm.streamsx.dps/impl/include/MongoDBLayer.h b/com.ibm.streamsx.dps/impl/include/MongoDBLayer.h index ae5ccc0..a41adc9 100644 --- a/com.ibm.streamsx.dps/impl/include/MongoDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/MongoDBLayer.h @@ -172,8 +172,11 @@ namespace distributed bool acquireLock(uint64_t lock, double leaseTime, double maxWaitTimeToAcquireLock, PersistenceError & lkError); bool removeLock(uint64_t lock, PersistenceError & lkError); uint32_t getPidForLock(std::string const & name, PersistenceError & lkError); - void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); }; } } } } } diff --git a/com.ibm.streamsx.dps/impl/include/RedisClusterDBLayer.h b/com.ibm.streamsx.dps/impl/include/RedisClusterDBLayer.h index a2d58f9..954f761 100644 --- a/com.ibm.streamsx.dps/impl/include/RedisClusterDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/RedisClusterDBLayer.h @@ -181,10 +181,13 @@ namespace distributed std::string const & baseUrl, std::string const & apiEndpoint, std::string const & queryParams, std::string const & jsonRequest, std::string & jsonResponse, PersistenceError & dbError); bool runDataStoreCommand(std::vector const & cmdList, std::string & resultValue, PersistenceError & dbError); - void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); - // Lock related methods. + // Lock related methods. uint64_t createOrGetLock(std::string const & name, PersistenceError & lkError); void releaseLock(uint64_t lock, PersistenceError & lkError); bool acquireLock(uint64_t lock, double leaseTime, double maxWaitTimeToAcquireLock, PersistenceError & lkError); diff --git a/com.ibm.streamsx.dps/impl/include/RedisClusterPlusPlusDBLayer.h b/com.ibm.streamsx.dps/impl/include/RedisClusterPlusPlusDBLayer.h index 930a98e..cc44124 100644 --- a/com.ibm.streamsx.dps/impl/include/RedisClusterPlusPlusDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/RedisClusterPlusPlusDBLayer.h @@ -156,10 +156,13 @@ namespace distributed std::string const & baseUrl, std::string const & apiEndpoint, std::string const & queryParams, std::string const & jsonRequest, std::string & jsonResponse, PersistenceError & dbError); bool runDataStoreCommand(std::vector const & cmdList, std::string & resultValue, PersistenceError & dbError); - void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); - // Lock related methods. + // Lock related methods. uint64_t createOrGetLock(std::string const & name, PersistenceError & lkError); void releaseLock(uint64_t lock, PersistenceError & lkError); bool acquireLock(uint64_t lock, double leaseTime, double maxWaitTimeToAcquireLock, PersistenceError & lkError); diff --git a/com.ibm.streamsx.dps/impl/include/RedisDBLayer.h b/com.ibm.streamsx.dps/impl/include/RedisDBLayer.h index fa78405..80d36b6 100644 --- a/com.ibm.streamsx.dps/impl/include/RedisDBLayer.h +++ b/com.ibm.streamsx.dps/impl/include/RedisDBLayer.h @@ -175,8 +175,11 @@ namespace distributed std::string const & baseUrl, std::string const & apiEndpoint, std::string const & queryParams, std::string const & jsonRequest, std::string & jsonResponse, PersistenceError & dbError); bool runDataStoreCommand(std::vector const & cmdList, std::string & resultValue, PersistenceError & dbError); - void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); - void getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error); + void getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError); + void getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError); + void putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError); + void hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError); + void removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError); // Lock related methods. uint64_t createOrGetLock(std::string const & name, PersistenceError & lkError); diff --git a/com.ibm.streamsx.dps/impl/src/CassandraDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/CassandraDBLayer.cpp index 7a13262..33139fa 100644 --- a/com.ibm.streamsx.dps/impl/src/CassandraDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/CassandraDBLayer.cpp @@ -2434,16 +2434,42 @@ namespace distributed } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of keys from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. - void CassandraDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "CassandraDBLayer"); + void CassandraDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "CassandraDBLayer"); // Not implemented at this time. Simply return. return; } + // Senthil added this on Apr/21/2022. + // This method will put the given K/V pairs in a given store. + void CassandraDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "CassandraDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + void CassandraDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "CassandraDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + void CassandraDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "CassandraDBLayer"); + + // Not implemented at this time. Simply return. + return; + } CassandraDBLayerIterator::CassandraDBLayerIterator() { diff --git a/com.ibm.streamsx.dps/impl/src/CloudantDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/CloudantDBLayer.cpp index 834dae0..77cf3ae 100644 --- a/com.ibm.streamsx.dps/impl/src/CloudantDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/CloudantDBLayer.cpp @@ -2308,11 +2308,38 @@ namespace distributed } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of keys from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. - void CloudantDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "CloudantDBLayer"); + void CloudantDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "CloudantDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/21/2022. + // This method will put the given K/V pairs in a given store. + void CloudantDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "CloudantDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + void CloudantDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "CloudantDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + void CloudantDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "CloudantDBLayer"); // Not implemented at this time. Simply return. return; diff --git a/com.ibm.streamsx.dps/impl/src/CouchbaseDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/CouchbaseDBLayer.cpp index 1b9c12a..4a2e7a6 100644 --- a/com.ibm.streamsx.dps/impl/src/CouchbaseDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/CouchbaseDBLayer.cpp @@ -3482,16 +3482,44 @@ namespace distributed } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of keys from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. - void CouchbaseDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "CouchbaseDBLayer"); + void CouchbaseDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "CouchbaseDBLayer"); // Not implemented at this time. Simply return. return; } + // Senthil added this on Apr/21/2022. + // This method will put the given K/V pairs in a given store. + void CouchbaseDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "CouchbaseDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + void CouchbaseDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "CouchbaseDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + void CouchbaseDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "CouchbaseDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + CouchbaseDBLayerIterator::CouchbaseDBLayerIterator() { } diff --git a/com.ibm.streamsx.dps/impl/src/HBaseDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/HBaseDBLayer.cpp index fae58cd..11d7ffa 100644 --- a/com.ibm.streamsx.dps/impl/src/HBaseDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/HBaseDBLayer.cpp @@ -2735,11 +2735,38 @@ namespace distributed } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of keya from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. - void HBaseDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "HBaseDBLayer"); + void HBaseDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "HBaseDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/21/2022. + // This method will put the given K/V pairs in a given store. + void HBaseDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "HBaseDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + void HBaseDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "HBaseDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + void HBaseDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "HBaseDBLayer"); // Not implemented at this time. Simply return. return; diff --git a/com.ibm.streamsx.dps/impl/src/MemcachedDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/MemcachedDBLayer.cpp index 6825012..292c9ae 100644 --- a/com.ibm.streamsx.dps/impl/src/MemcachedDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/MemcachedDBLayer.cpp @@ -2287,11 +2287,38 @@ namespace distributed } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of keys from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. - void MemcachedDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "MemcachedDBLayer"); + void MemcachedDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "MemcachedDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/21/2022. + // This method will put the given K/V pairs in a given store. + void MemcachedDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "MemcachedDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + void MemcachedDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "MemcachedDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + void MemcachedDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "MemcachedDBLayer"); // Not implemented at this time. Simply return. return; diff --git a/com.ibm.streamsx.dps/impl/src/MongoDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/MongoDBLayer.cpp index 34b0c4e..ebf82e7 100644 --- a/com.ibm.streamsx.dps/impl/src/MongoDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/MongoDBLayer.cpp @@ -2101,11 +2101,38 @@ namespace distributed } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of keys from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. - void MongoDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "MongoDBLayer"); + void MongoDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "MongoDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/21/2022. + // This method will put the given K/V pairs in a given store. + void MongoDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "MongoDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + void MongoDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "MongoDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + void MongoDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "MongoDBLayer"); // Not implemented at this time. Simply return. return; diff --git a/com.ibm.streamsx.dps/impl/src/RedisClusterDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/RedisClusterDBLayer.cpp index 119b21a..5b6c654 100644 --- a/com.ibm.streamsx.dps/impl/src/RedisClusterDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/RedisClusterDBLayer.cpp @@ -2747,11 +2747,38 @@ namespace distributed } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of key from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. - void RedisClusterDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "RedisClusterDBLayer"); + void RedisClusterDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "RedisClusterDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/21/2022. + // This method will put the given K/V pairs in a given store. + void RedisClusterDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "RedisClusterDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + void RedisClusterDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "RedisClusterDBLayer"); + + // Not implemented at this time. Simply return. + return; + } + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + void RedisClusterDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "RedisClusterDBLayer"); // Not implemented at this time. Simply return. return; diff --git a/com.ibm.streamsx.dps/impl/src/RedisClusterPlusPlusDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/RedisClusterPlusPlusDBLayer.cpp index b3e6a69..a79015e 100644 --- a/com.ibm.streamsx.dps/impl/src/RedisClusterPlusPlusDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/RedisClusterPlusPlusDBLayer.cpp @@ -1274,7 +1274,7 @@ namespace distributed } // Apr/16/2022 - // We may have a Redis Sorted Set associated with this store being removed. That set is used for bulk APIs. We can remove it now as the store itself is gone now. + // We will have a Redis Sorted Set associated with this store being removed. That set is used for bulk APIs. We can remove it now as the store itself is gone now. string keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; exceptionString = ""; exceptionType = REDIS_PLUS_PLUS_NO_ERROR; @@ -1363,6 +1363,11 @@ namespace distributed // Ideally, at this point here in this API, we can check for whether the store exists or not. But, that will slow down this // faster put API. Hence, we are going to trust the users to call this faster API only on existing stores. + // Adding a key will be a two step process by doing both HSET and ZADD one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // // In our Redis dps implementation, data item keys can have space characters. string data_item_key = string(keyData, keySize); @@ -1429,6 +1434,8 @@ namespace distributed return(false); } + // Step 2: + // // Apr/15/2022. // We will also add the key we stored above in a Redis sorted set. That will be useful for // implementing bulk APIs (donw in later part of this source file) to get a range of keys. @@ -1438,7 +1445,7 @@ namespace distributed // We will add the key involved in this put to this store's sorted set. // NX flag in this command means that only add new elements. Don't update already existing elements. - // All the keys in our store's sorted set will have a random scoreto make the sorting work faster. + // All the keys in our store's sorted set will have a random score to make the sorting work faster. // // Generate a random number by using the C++11 supported APIs. std::default_random_engine generator; @@ -1541,6 +1548,11 @@ namespace distributed return(false); } + // Adding a key will be a two step process by doing both HSET and ZADD one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // // We are ready to either store a new data item or update an existing data item. // This action is performed on the Store Contents Hash that takes the following format. // '1' + 'store id' => 'Redis Hash' [It will always have this entry: dps_name_of_this_store ==> 'store name'] @@ -1607,6 +1619,8 @@ namespace distributed return(false); } + // Step 2: + // // Apr/15/2022. // We will also add the key we stored above in a Redis sorted set. That will be useful for // implementing bulk APIs (donw in later part of this source file) to get a range of keys. @@ -2011,6 +2025,11 @@ namespace distributed return(false); } + // Removing a key will be a two step process by doing both HDEL and ZREM one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // // This action is performed on the Store Contents Hash that takes the following format. // '1' + 'store id' => 'Redis Hash' [It will always have this entry: dps_name_of_this_store ==> 'store name'] string keyString = string(DPS_STORE_CONTENTS_HASH_TYPE) + storeIdString; @@ -2082,6 +2101,8 @@ namespace distributed return(false); } + // Step 2: + // // Apr/16/2022 // I recently added support for bulk APIs. For that, I had to use a sorted set and keep all the // store keys in sorted order. Since we removed a store key above, it is necessary to remove that @@ -3422,7 +3443,7 @@ namespace distributed // NOTE1: In Redis, there are a few commands that take no arguments. // Such commands can't be executed via the generic command API. // - // NOTE2: Certain Redis command support the use of multiple keys. + // NOTE2: Certain Redis commands support the use of multiple keys. // Because of that, there is a potential for CROSSSLOT error. // This code block can handle multiple keys in a single command. // However, it will give an exception as shown below if there are no @@ -3952,7 +3973,7 @@ namespace distributed /// @param keys User provided mutable list variable. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. /// @param keyStartPosition User can indicate a start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. /// @param numberOfKeysNeeded User can indicate the total number of keys to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available keys upto a maximum of 50000 keys from the given key start position will be returned. - /// @param err Contains the error code. Will be '0' if no error occurs, and a non-zero value otherwise. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// void RedisClusterPlusPlusDBLayer::getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError) { SPLAPPTRC(L_DEBUG, "Inside getKeys for store id " << store, "RedisClusterPlusPlusDBLayer"); @@ -4037,11 +4058,11 @@ namespace distributed // This is how we can detect that a wrong redis-cluster server name is configured by the user or // not even a single redis-cluster server daemon being up and running. // This is a serious error. - dbError.set(string("getKeys: Unable to connect to the redis-cluster server(s).") + + dbError.set(string("Inside getKeys #1, unable to connect to the redis-cluster server(s).") + string(" Got an exception for REDIS_ZRANGE_CMD: ") + exceptionString + " Application code may call the DPS reconnect API and then retry the failed operation. ", DPS_CONNECTION_ERROR); - SPLAPPTRC(L_ERROR, "Inside getKeys, it failed with a Redis connection error for REDIS_ZRANGE_CMD. Exception: " << exceptionString << ". Application code may call the DPS reconnect API and then retry the failed operation. " << DPS_CONNECTION_ERROR, "RedisClusterPlusPlusDBLayer"); + SPLAPPTRC(L_ERROR, "Inside getKeys #1, , it failed with a Redis connection error for REDIS_ZRANGE_CMD. Exception: " << exceptionString << ". Application code may call the DPS reconnect API and then retry the failed operation. " << DPS_CONNECTION_ERROR, "RedisClusterPlusPlusDBLayer"); return; } @@ -4049,8 +4070,8 @@ namespace distributed if (exceptionType == REDIS_PLUS_PLUS_REPLY_ERROR || exceptionType == REDIS_PLUS_PLUS_OTHER_ERROR) { // Unable to get data item keys from the store. - dbError.set("Unable to get keys via ZRANGE for the StoreId " + storeIdString + ". Error=" + exceptionString, DPS_GET_STORE_DATA_ITEM_KEYS_ERROR); - SPLAPPTRC(L_ERROR, "Inside getKeys, ZRANGE failed to get keys from the StoreId " << storeIdString << ". Error=" << exceptionString << ". rc=" << DPS_GET_STORE_DATA_ITEM_KEYS_ERROR, "RedisClusterPlusPlusDBLayer"); + dbError.set("Inside getKeys #2, unable to get keys via ZRANGE for the StoreId " + storeIdString + ". Error=" + exceptionString, DPS_GET_STORE_DATA_ITEM_KEYS_ERROR); + SPLAPPTRC(L_ERROR, "Inside getKeys #2, ZRANGE failed to get keys from the StoreId " << storeIdString << ". Error=" << exceptionString << ". rc=" << DPS_GET_STORE_DATA_ITEM_KEYS_ERROR, "RedisClusterPlusPlusDBLayer"); return; } @@ -4073,8 +4094,8 @@ namespace distributed // We will leave any partially filled data in the user provided list. // On detection the db error flag, caller should take the necessary steps to // free the allocated memory for the partially filled data in that list. - dbError.set("Unable to allocate memory for the keyData while fetching bulk keys in getKeys for the StoreId " + storeIdString + ".", DPS_STORE_ITERATION_MALLOC_ERROR); - SPLAPPTRC(L_DEBUG, "Inside getKeys, it failed for store id " << storeIdString << ". " << DPS_STORE_ITERATION_MALLOC_ERROR, "RedisClusterPlusPlusDBLayerr"); + dbError.set("Inside getKeys #3, unable to allocate memory for the keyData while fetching bulk keys in getKeys for the StoreId " + storeIdString + ".", DPS_STORE_ITERATION_MALLOC_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getKeys #3, it failed for store id " << storeIdString << ". " << DPS_STORE_ITERATION_MALLOC_ERROR, "RedisClusterPlusPlusDBLayerr"); return; } @@ -4088,45 +4109,277 @@ namespace distributed } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of keys from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. /// /// @param store The handle of the store given in a string form. /// @param key User provided key for which a value needs to be fetched. - /// @param keySixe Size of the buffer holding the key. + /// @param keySize Size of the buffer holding the key. /// @param value Buffer pointer where the fetched value will be made available for the caller. /// @param valueSize Size of the buffer holding the value. - /// @param error Error code from the get operation if any will be made available for the caller. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get values operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// - void RedisClusterPlusPlusDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "RedisClusterPlusPlusDBLayer"); + void RedisClusterPlusPlusDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "RedisClusterPlusPlusDBLayer"); + + // Create a string form of the store id. + std::ostringstream storeId; + storeId << store; + std::string storeIdString = storeId.str(); // This method will skip all the safety checks such as store existence, key existence etc. // If store safety is needed, the caller of this API can do their own DPS locks at the application level. // They can also check for key existence before sending that key to be used here for a value fetch. string keyString = string(DPS_STORE_CONTENTS_HASH_TYPE) + storeIdString; - value = NULL; - valueSize = 0; - string exceptionString = ""; int exceptionType = REDIS_PLUS_PLUS_NO_ERROR; - string data_item_value = ""; + + // NOTE: Since a given key might NOT exist in the store, so we need to parse it as OptionalString. + // That data type can support Redis Nil value objects for non-existing keys. + std::vector values; // In our Redis dps implementation, data item keys can have space characters. // Original NBF serialized key is stored in Redis as a base64 encoded string. // So, to query, we will have to pass a base64 encoded key. - string base64_encoded_data_item_key; - base64_encode(string(key, keySize), base64_encoded_data_item_key); + // We should have all the base64 encoded keys in their own memory locations instead of + // keep reusing a single local variable. + std::vector base64_keys; + + // Initialize the base64 keys vector with empty string elements. + for(int i=0; i<(int)keyData.size(); i++) { + base64_keys.push_back(std::string("")); + } + // We are given all the keys in a separate list. + // That list is holding the NBF formatted buffers for the keys. + // We can iterate over that list and keep populating two other lists required for + // executing the variadic version of the Redis command API. For binary safe data, + // we have to populate the size of every data item in the argvlen vector. + for(int i=0; i<(int)keyData.size(); i++) { + // Base64 encode a key. + base64_encode(string(keyData.at(i), keySize.at(i)), base64_keys.at(i)); + } + + // We will use the Redis variadic API to do multiple gets in one call to that API. // Fetch the data item now. try { - auto my_value = redis_cluster->hget(keyString, base64_encoded_data_item_key); + redis_cluster->hmget(keyString, base64_keys.begin(), base64_keys.end(), std::back_inserter(values)); + } catch (const ReplyError &ex) { + // WRONGTYPE Operation against a key holding the wrong kind of value + exceptionString = ex.what(); + // Command execution error. + exceptionType = REDIS_PLUS_PLUS_REPLY_ERROR; + } catch (const TimeoutError &ex) { + // Reading or writing timeout + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const ClosedError &ex) { + // Connection has been closed. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const IoError &ex) { + // I/O error on the connection. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const Error &ex) { + // Other errors + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_OTHER_ERROR; + } - if(my_value) { - data_item_value = *my_value; - } + // Did we encounter a redis-cluster server connection error? + if (exceptionType == REDIS_PLUS_PLUS_CONNECTION_ERROR) { + dbError.set(std::string("Inside getValues #1, unable to connect to the redis cluster server(s). Exception=") + exceptionString, DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getValues #1, it failed for executing the HMGET command for multiple keys. Exception=" << exceptionString << ". " << DPS_CONNECTION_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + + // Did we encounter a redis reply error? + if (exceptionType == REDIS_PLUS_PLUS_REPLY_ERROR || + exceptionType == REDIS_PLUS_PLUS_OTHER_ERROR) { + // Problem in reading data items from the cache. + dbError.set(std::string("Inside getValues #2, unable to get data items via HMGET. Exception=") + exceptionString, DPS_BULK_GET_HMGET_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getValues #2, it failed to get data items via HMGET. Exception=" << exceptionString << ". " << DPS_BULK_GET_HMGET_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + + // This Redis varidatic API will return an array of results i.e. values. + // In an array based result, there may be Redis nil result items if the key doesn't exist. + // So, it is safe to use the OptionalString instead of std::string which can lead to an exception while trying to convert nil to std::string. + for(int i=0; i<(int)values.size(); i++) { + OptionalString s = values.at(i); + + if(s) { + // Consider this result item only if the OptionalString really has a non-zero length value. + uint32_t myValueSize = (*s).length(); + + if(myValueSize <= 0) { + // We got a zero length data item value. + // This is not very useful and it should never happen. + // We will raise an error for this. + dbError.set(std::string("Inside getValues #3, Found an empty data item value for the storeId ") + storeIdString, DPS_BULK_GET_HMGET_EMPTY_VALUE_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getValues #3: Found an empty data item value for the StoreId " << storeIdString << ".", "RedisClusterPlusPlusDBLayer"); + return; + } else { + // We can allocate memory for the exact length of the data item value. + unsigned char * value = (unsigned char *) malloc(myValueSize); + + if (value == NULL) { + dbError.set(std::string("Inside getValues #4, unable to allocate memory to copy the data item value for the storeId ") + storeIdString, DPS_BULK_GET_HMGET_MALLOC_ERROR); + // Unable to allocate memory to transfer the data item value. + SPLAPPTRC(L_DEBUG, "Inside getValues #4: Unable to allocate memory to copy the data item value for the StoreId " << storeIdString << ".", "RedisClusterPlusPlusDBLayer"); + return; + } else { + // We expect the caller of this method to free the value pointer. + memcpy(value, (*s).c_str(), myValueSize); + // We can add it to the user provided lists. + valueData.push_back(value); + valueSize.push_back(myValueSize); + } + } + } else { + // This Redis result probably is Nil. + SPLAPPTRC(L_DEBUG, "Inside getValues #5, it returned a NULL result for key index " << i, "RedisClusterPlusPlusDBLayer"); + valueData.push_back(NULL); + valueSize.push_back(0); + continue; + } + } // End of for loop. + } // End of getValues method. + + // Senthil added this on Apr/24/2022. + // This method will put the given K/V pairs in a given store. + // It is a bulk put operation. + void RedisClusterPlusPlusDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "RedisClusterPlusPlusDBLayer"); + + // Get the string form of the store id. + std::ostringstream storeId; + storeId << store; + string storeIdString = storeId.str(); + + // Adding keys will be a two step process by doing both HSET and ZADD one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // + string keyString = string(DPS_STORE_CONTENTS_HASH_TYPE) + storeIdString; + string exceptionString = ""; + int exceptionType = REDIS_PLUS_PLUS_NO_ERROR; + + // We should have all the base64 encoded keys in their own memory locations instead of + // keep reusing a single local variable. + std::vector base64_keys; + + // Initialize the base64 keys vector with empty string elements. + for(int i=0; i<(int)keyData.size(); i++) { + base64_keys.push_back(std::string("")); + } + + // Have another vector for the values. + std::vector values; + + // In Redis cluster plus plus, bulk put works very differently from what we have done in hiredis standalone driver. + // We can create an unordered map of all the keys and values and then issue a single call to the hset API. + std::unordered_map kvPairs; + + // We are given all the keys and the corresponding values in two separate lists. + // Those two lists are holding the NBF formatted buffers for the keys and values. + // We can iterate over the lists and keep populating the unordered map. + for(int i=0; i<(int)keyData.size(); i++) { + // Base64 encode a key. + base64_encode(string(keyData.at(i), keySize.at(i)), base64_keys.at(i)); + // Stringify the value data and keep it in the vector we declared above. + values.push_back(string((const char*)valueData.at(i), (size_t)valueSize.at(i))); + + // Now put the K/V pair in the unordered map. + kvPairs.insert({base64_keys.at(i), values.at(i)}); + } + + // Let us do a hset of multiple K/V pairs. + try { + redis_cluster->hset(keyString, kvPairs.begin(), kvPairs.end()); + } catch (const ReplyError &ex) { + // WRONGTYPE Operation against a key holding the wrong kind of value + exceptionString = ex.what(); + // Command execution error. + exceptionType = REDIS_PLUS_PLUS_REPLY_ERROR; + } catch (const TimeoutError &ex) { + // Reading or writing timeout + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const ClosedError &ex) { + // Connection has been closed. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const IoError &ex) { + // I/O error on the connection. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const Error &ex) { + // Other errors + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_OTHER_ERROR; + } + + // Did we encounter a redis-cluster server connection error? + if (exceptionType == REDIS_PLUS_PLUS_CONNECTION_ERROR) { + dbError.set(std::string("Inside putKVPairs #1, unable to connect to the redis cluster server(s). Exception=") + exceptionString, DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside putKVPairs #1, it failed with a Redis connection error for executing the hset command with multiple K/V pairs. Exception=" << exceptionString << ". Application code may call the DPS reconnect API and then retry the failed operation. " << DPS_CONNECTION_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + + // Did we encounter a redis reply error? + if (exceptionType == REDIS_PLUS_PLUS_REPLY_ERROR || + exceptionType == REDIS_PLUS_PLUS_OTHER_ERROR) { + // Problem in storing a data item in the cache. + dbError.set(std::string("Inside putKVPairs #2, unable to store a data item. Exception=") + exceptionString, DPS_BULK_PUT_HSET_ERROR); + SPLAPPTRC(L_DEBUG, "Inside putKVPairs #2, it failed to store a data item to the StoreId " << storeIdString << ". Exception=" << exceptionString << ". " << DPS_BULK_PUT_HSET_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + + // Step 2: + // + // Now that we have put multiple K/V Pairs, let us now add the keys for them in + // zset (Sorted Set) that we keep for every DPS store. That will be useful for + // implementing bulk APIs to get a range of keys. + // This action is performed on the Store Ordered Keys Set that takes the following format. + // '101' + 'store id' => 'Redis SortedSet' + keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; + exceptionString = ""; + exceptionType = REDIS_PLUS_PLUS_NO_ERROR; + + // This map will hold member:score pairs for the zset. + std::unordered_map scoreAndMember; + std::vector scores; + // Generate a random number by using the C++11 supported APIs. + std::default_random_engine generator; + // Random number range is as shown here. + std::uniform_int_distribution distribution(1, 5000000); + + // Stay in a loop and populate the map with the required score and member data. + for(int i=0; i<(int)keyData.size(); i++) { + scores.push_back(distribution(generator)); + // Now put the Score/Member pair in the unordered map. + scoreAndMember.insert({base64_keys.at(i), scores.at(i)}); + } + + // We will add the key involved in this put to this store's sorted set. + // NX flag in this command means that only add new elements. Don't update already existing elements. + // All the keys in our store's sorted set will have a random score to make the sorting work faster. + try { + // This should do a bulk add to the sorted set. + redis_cluster->zadd(keyString, scoreAndMember.begin(), scoreAndMember.end(), UpdateType::NOT_EXIST); } catch (const ReplyError &ex) { // WRONGTYPE Operation against a key holding the wrong kind of value exceptionString = ex.what(); @@ -4156,48 +4409,428 @@ namespace distributed // Did we encounter a redis-cluster server connection error? if (exceptionType == REDIS_PLUS_PLUS_CONNECTION_ERROR) { - // This is how we can detect that a wrong redis-cluster server name is configured by the user or - // not even a single redis-cluster server daemon being up and running. - // This is a serious error. - SPLAPPTRC(L_DEBUG, "Inside getValue #1: It failed with a Redis connection error for REDIS_HGET_CMD. Exception: " << exceptionString << ". Application code may call the DPS reconnect API and then retry the failed operation. " << DPS_CONNECTION_ERROR, "RedisClusterPlusPlusDBLayer"); - error = DPS_CONNECTION_ERROR; + dbError.set(string("Inside putKVPairs #3: Unable to connect to the redis-cluster server(s).") + + string(" Got an exception for REDIS_ZADD_CMD: ") + exceptionString + + " Application code may call the DPS reconnect API and then retry the failed operation. ", + DPS_CONNECTION_ERROR); + SPLAPPTRC(L_ERROR, "Inside putKVPairs #3: It failed for store id " << storeIdString << " with a Redis connection error for REDIS_ZADD_CMD. Exception: " << exceptionString << ". Application code may call the DPS reconnect API and then retry the failed operation. " << DPS_CONNECTION_ERROR, "RedisClusterPlusPlusDBLayer"); return; } // Did we encounter a redis reply error? if (exceptionType == REDIS_PLUS_PLUS_REPLY_ERROR || exceptionType == REDIS_PLUS_PLUS_OTHER_ERROR) { - SPLAPPTRC(L_DEBUG, "Inside getValue #2: It failed to get a data item from the StoreId " << storeIdString << ". Error=" << exceptionString << ". rc=" << DPS_DATA_ITEM_READ_ERROR, "RedisClusterPlusPlusDBLayer"); - error = DPS_REDIS_REPLY_NULL_ERROR; + dbError.set("Inside putKVPairs #4: Unable to do bulk ZADD in store's sorted set for the store id " + storeIdString + + ". Error=" + exceptionString, DPS_BULK_PUT_ZADD_ERROR); + SPLAPPTRC(L_ERROR, "Inside putKVPairs #4: It failed for store id " << storeIdString << " while doing bulk ZADD in store's sorted set. Error=" << exceptionString << ". rc=" << DPS_BULK_PUT_ZADD_ERROR, "RedisClusterPlusPlusDBLayer"); return; } + } // End of putKVPairs method. + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + // This is a bulk key existence check operation. + void RedisClusterPlusPlusDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "RedisClusterPlusPlusDBLayer"); + + // Get the string form of the store id. + std::ostringstream storeId; + storeId << store; + string storeIdString = storeId.str(); + + // To check the existence of multiple keys in a store via a single API call, + // Redis doesn't have a straightforward command that can be executed. + // In fact, until the Redis v6.2.0 release, it was impossible to do. In v6.2.0, + // they introduced ZMSCORE to check multiple members' scores in a sorted set. + // Since we recently introduced in DPS to shadow every store hash with a zset to + // support some of the bulk APIs, we are going to rely on the new ZMSCORE + // redis command to check existence of multiple keys. + // Because of the use of ZMSCORE, DPS toolkit will require Redis v6.2.0 or a + // higher version going forward from this date (Apr/27/2022). + // + string keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; + + // In the Redis Plus Plus C++ driver, there is no direct API available for + // zmscore as of Apr/27/2022. Until such a direct API is availble in that + // driver, we have to use the genric command support that driver provides. + // This approach may have a slight performance overhead compared to a direct API. + // We have to simply use it for now until a good option becomes available at a later time. + // + // Declare a vector to keep collecting the individual parts of the + // ZMSCORE command. + std::vector cmdList; + + // Prepare to run the Redis ZMSCORE command. + string zmscoreCommand = string(REDIS_ZMSCORE_CMD); + // Strip the space at the end of the command that should not be there in the cmdList vector.. + zmscoreCommand = zmscoreCommand.substr(0, zmscoreCommand.size()-1); + + // Let us push the zmscore command string. + cmdList.push_back(zmscoreCommand); + // Push the keystring. + cmdList.push_back(keyString); + + // We should have all the base64 encoded keys in their own memory locations instead of + // keep reusing a single local variable. + std::vector base64_keys; + + // Initialize the base64 keys vector with empty string elements. + for(int i=0; i<(int)keyData.size(); i++) { + base64_keys.push_back(std::string("")); + } - // Data item value read from the store will be in this format: 'value' - valueSize = data_item_value.length(); + // We are given all the keys in a separate list. + // That list is holding the NBF formatted buffers for the keys. + // We can iterate over that list and keep populating all the + // zset members to the command vector that we created above. + for(int i=0; i<(int)keyData.size(); i++) { + // Base64 encode a key. + base64_encode(string(keyData.at(i), keySize.at(i)), base64_keys.at(i)); + // Push a key now. + cmdList.push_back(base64_keys.at(i)); + } - if (valueSize > 0) { - // We can allocate memory for the exact length of the data item value. - value = (unsigned char *) malloc(valueSize); + string exceptionString = ""; + int exceptionType = REDIS_PLUS_PLUS_NO_ERROR; - if (value == NULL) { - valueSize = 0; - error = DPS_GET_DATA_ITEM_MALLOC_ERROR; - // Unable to allocate memory to transfer the data item value. - SPLAPPTRC(L_DEBUG, "Inside getValue #5: Unable to allocate memory to copy the data item value for the StoreId " << storeIdString << ".", "RedisClusterPlusPlusDBLayer"); - } else { - // We expect the caller of this method to free the value pointer. - memcpy(value, data_item_value.c_str(), valueSize); - error = 0; + try { + // Execute it using the generic command API. + auto r = redis_cluster->command(cmdList.begin(), cmdList.end()); + + /* + Redis can return one of these replies after executing a command: + + Status Reply: Also known as Simple String Reply. It's a non-binary string reply. + Bulk String Reply: Binary safe string reply. + Integer Reply: Signed integer reply. Large enough to hold long long. + Array Reply: (Nested) Array reply. + Error Reply: Non-binary string reply that gives error info. + */ + if(reply::is_error(*r) == true) { + // Problem reading scores for the zset members. + dbError.set(std::string("Inside hasKeys #1, unable to get scores for the given keys. "), DPS_BULK_GET_ZMSCORE_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #1, it failed to get scores for the given keys. " << DPS_BULK_GET_ZMSCORE_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + + // We expect a Redis reply array as a result type in this case. + if(reply::is_array(*r) == false) { + // Problem in getting multi value result as an array result type.. + dbError.set(std::string("Inside hasKeys #2, unable to get the scores in an array. "), DPS_BULK_GET_ZMSCORE_NO_REPLY_ARRAY_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #2, it failed to get the scores in an array. " << DPS_BULK_GET_ZMSCORE_NO_REPLY_ARRAY_ERROR, "RedisClusterPlusPlusDBLayer"); + return; } - } else { - // We got a zero length data item value. - // This is not very useful and it should never happen. - // We will raise an error for this. - valueSize = 0; - error = DPS_EMPTY_DATA_ITEM_VALUE_FOUND_ERROR; - SPLAPPTRC(L_DEBUG, "Inside getValue #3: Found an empty data item value for the StoreId " << storeIdString << ".", "RedisClusterPlusPlusDBLayer"); - } // End of if (valueSize > 0) - } // End of getValue method. + + // We have the scores for the given keys returned in a Redis array now. + // Non-existing keys will result in Redis Nil. + // Let us start collecting them as boolean true or false to be sent back to the caller. + // In an array based result, there may be Redis nil result items if the query didn't yield any valid result. + // So, it is safe to use the OptionalString instead of std::string which can lead to an exception while trying to convert nil to std::string. + std::vector scoresResult = reply::parse>(*r); + + // Iterate over all the zset score results we got back from Redis. + for(uint32_t i = 0; i < scoresResult.size(); i++) { + OptionalString s = scoresResult.at(i); + + if(s) { + // Consider this result item only if the OptionalString really has a non-zero length value. + uint32_t myValueSize = (*s).length(); + + // Score read from the sorted set will be in this format: 'value' + if (myValueSize > 0) { + // This is a valid score. That means the given key exists. + results.push_back(true); + } else { + // We got a zero length score. + // This is not very useful and it should never happen. + // We will raise an error for this. + dbError.set(std::string("Inside hasKeys #3, Found an empty score in zset for the storeId ") + storeIdString, DPS_BULK_GET_ZMSCORE_EMPTY_VALUE_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #3: Found an empty score in zset for the StoreId " << storeIdString << ".", "RedisClusterPlusPlusDBLayer"); + return; + } + } else { + // This Redis result probably is Nil. + SPLAPPTRC(L_DEBUG, "Inside hasKeys #4, it returned a NULL result for key index " << i, "RedisClusterPlusPlusDBLayer"); + // This means the given key doesn't exist in the sorted set i.e. in the DPS store hash as well since the + // sorted set mirrors our store hash at all times. + results.push_back(false); + continue; + } + } // End of for loop. + } catch (const ReplyError &ex) { + // WRONGTYPE Operation against a key holding the wrong kind of value + exceptionString = ex.what(); + // Command execution error. + exceptionType = REDIS_PLUS_PLUS_REPLY_ERROR; + } catch (const TimeoutError &ex) { + // Reading or writing timeout + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const ClosedError &ex) { + // Connection has been closed. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const IoError &ex) { + // I/O error on the connection. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const Error &ex) { + // Other errors + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_OTHER_ERROR; + } + + if(exceptionType != REDIS_PLUS_PLUS_NO_ERROR) { + // Problem reading scores for the zset members. + dbError.set(std::string("Inside hasKeys #5, unable to get scores for the given keys. Exception=") + exceptionString, DPS_BULK_GET_ZMSCORE_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #5, it failed to get scores for the given keys. Exceptipn=" << exceptionString << ". " << DPS_BULK_GET_ZMSCORE_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + } // End of hasKeys method. + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + // This is a bulk key removal operation. + void RedisClusterPlusPlusDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "RedisClusterPlusPlusDBLayer"); + + // Get the string form of the store id. + std::ostringstream storeId; + storeId << store; + string storeIdString = storeId.str(); + + string keyString = string(DPS_STORE_CONTENTS_HASH_TYPE) + storeIdString; + + // Removing keys will be a two step process by doing both HDEL and ZREM one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // In the Redis plus plus driver, they have hdel_range and zrem_range for + // bulk key removals. However, those APIs don't return back the total + // number of keys removed. We have to return that value to the caller. + // Because of that, we can't use those two APIs. Instead, we will use the + // generic command supported by that driver to form our own HDEL and ZREM + // command strings with multiple keys. + // + // Declare a vector to keep collecting the individual parts of the + // HDEL command. + std::vector cmdList; + + // Step 1: + // + // Prepare to run the Redis HDEL command. + string hdelCommand = string(REDIS_HDEL_CMD); + // Strip the space at the end of the command that should not be there in the cmdList vector. + hdelCommand = hdelCommand.substr(0, hdelCommand.size()-1); + + // Let us push the hdel command string. + cmdList.push_back(hdelCommand); + // Push the keystring. + cmdList.push_back(keyString); + + // We should have all the base64 encoded keys in their own memory locations instead of + // keep reusing a single local variable. + std::vector base64_keys; + + // Initialize the base64 keys vector with empty string elements. + for(int i=0; i<(int)keyData.size(); i++) { + base64_keys.push_back(std::string("")); + } + + // We are given all the keys in a separate list. + // That list is holding the NBF formatted buffers for the keys. + // We can iterate over that list and keep populating all the + // keys to be removed to the command vector that we created above. + for(int i=0; i<(int)keyData.size(); i++) { + // Base64 encode a key. + base64_encode(string(keyData.at(i), keySize.at(i)), base64_keys.at(i)); + // Push a key now. + cmdList.push_back(base64_keys.at(i)); + } + + string exceptionString = ""; + int exceptionType = REDIS_PLUS_PLUS_NO_ERROR; + + // Let us do a hdel of multiple keys from the given store. + try { + // Execute it using the generic command API. + auto r = redis_cluster->command(cmdList.begin(), cmdList.end()); + + /* + Redis can return one of these replies after executing a command: + + Status Reply: Also known as Simple String Reply. It's a non-binary string reply. + Bulk String Reply: Binary safe string reply. + Integer Reply: Signed integer reply. Large enough to hold long long. + Array Reply: (Nested) Array reply. + Error Reply: Non-binary string reply that gives error info. + */ + if(reply::is_error(*r) == true) { + // Problem removing the keys from the store hash. + dbError.set(std::string("Inside removeKeys #1, unable to remove multiple keys from a given store hash."), DPS_BULK_REMOVE_HDEL_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #1, it failed to remove multiple keys from a given store hash. " << DPS_BULK_REMOVE_HDEL_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + + // We expect a Redis integer result type in this case. + if(reply::is_integer(*r) == false) { + // Problem in getting HDEL result reply as an integer value. + dbError.set(std::string("Inside removeKeys #2, unable to get the keys removed cnt as an integer."), DPS_BULK_REMOVE_HDEL_NO_INTEGER_REPLY_ERROR ); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #2, it failed to get the keys removed cnt as an integer. " << DPS_BULK_REMOVE_HDEL_NO_INTEGER_REPLY_ERROR , "RedisClusterPlusPlusDBLayer"); + return; + } + + // If we are here, that means we have an integer reply that tells us + // as how many keys in total were removed. + auto num1 = reply::parse(*r); + totalKeysRemoved = num1; + } catch (const ReplyError &ex) { + // WRONGTYPE Operation against a key holding the wrong kind of value + exceptionString = ex.what(); + // Command execution error. + exceptionType = REDIS_PLUS_PLUS_REPLY_ERROR; + } catch (const TimeoutError &ex) { + // Reading or writing timeout + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const ClosedError &ex) { + // Connection has been closed. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const IoError &ex) { + // I/O error on the connection. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const Error &ex) { + // Other errors + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_OTHER_ERROR; + } + + if(exceptionType != REDIS_PLUS_PLUS_NO_ERROR) { + // Problem removing keys from the store hash. + dbError.set(std::string("Inside removeKeys #3, unable to remove multiple keys from a given store hash. Exception=" + exceptionString), DPS_BULK_REMOVE_HDEL_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #3, it failed to remove multiple keys from a given store hash. Exception=" << exceptionString << ". " << DPS_BULK_REMOVE_HDEL_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + + // Step 2: + // + // Now that we have removed multiple keys from the store, let us now remove their + // corresponding members that we keep in a zset (Sorted Set) for every DPS store. + // This action is performed on the Store Ordered Keys Set that takes the following format. + // '101' + 'store id' => 'Redis SortedSet' + keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; + // Clear the command list. + cmdList.clear(); + + // Prepare to run the Redis ZREM command. + string zremCommand = string(REDIS_ZREM_CMD); + // Strip the space at the end of the command that should not be there in the cmdList vector.. + zremCommand = zremCommand.substr(0, zremCommand.size()-1); + + // Let us push the zrem command string. + cmdList.push_back(zremCommand); + // Push the keystring. + cmdList.push_back(keyString); + + // We already have the zset members as we added them to a vector in the previous section. + // We can iterate over that list and keep populating all the + // members to be removed to the command vector that we created above. + for(int i=0; i<(int)keyData.size(); i++) { + // Push a zset member now. + cmdList.push_back(base64_keys.at(i)); + } + + exceptionString = ""; + exceptionType = REDIS_PLUS_PLUS_NO_ERROR; + int32_t total_zrem_cnt = 0; + + // Let us do a zrem of multiple members from the given zset. + try { + // Execute it using the generic command API. + auto r = redis_cluster->command(cmdList.begin(), cmdList.end()); + + /* + Redis can return one of these replies after executing a command: + + Status Reply: Also known as Simple String Reply. It's a non-binary string reply. + Bulk String Reply: Binary safe string reply. + Integer Reply: Signed integer reply. Large enough to hold long long. + Array Reply: (Nested) Array reply. + Error Reply: Non-binary string reply that gives error info. + */ + if(reply::is_error(*r) == true) { + // Problem removing the members from the store zset. + dbError.set(std::string("Inside removeKeys #4, unable to remove multiple members from a given zset."), DPS_BULK_REMOVE_ZREM_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #4, it failed to remove multiple members from a given zset. " << DPS_BULK_REMOVE_ZREM_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + + // We expect a Redis integer result type in this case. + if(reply::is_integer(*r) == false) { + // Problem in getting ZREM result reply as an integer value. + dbError.set(std::string("Inside removeKeys #5, unable to get members removed count as an integer reply."), DPS_BULK_REMOVE_ZREM_NO_INTEGER_REPLY_ERROR ); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #5, it failed to get members removed count as an integer reply. " << DPS_BULK_REMOVE_ZREM_NO_INTEGER_REPLY_ERROR , "RedisClusterPlusPlusDBLayer"); + return; + } + + // If we are here, that means we have an integer reply that tells us + // as how many zset members in total were removed. + // If we are here, that means we have an integer reply. + auto num2 = reply::parse(*r); + total_zrem_cnt = num2; + + // We have to check that we removed the same number of keys in both the + // store hash and store zset. If they are not the same number, it is a serious error that manifested + // either in this method or elsewhere and it will cause key, member imbalance and problems later. + if(totalKeysRemoved != total_zrem_cnt) { + // Remove cnt mismatch between the store and the zset. + dbError.set(std::string("Inside removeKeys #7, key, member removal cnt mismatch between HDEL and ZREM."), DPS_BULK_REMOVE_CNT_MISMATCH_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #7, it failed with a key, member removal cnt mismatch between HDEL and ZREM. " << DPS_BULK_REMOVE_CNT_MISMATCH_ERROR, "RedisDBLayer"); + return; + } + } catch (const ReplyError &ex) { + // WRONGTYPE Operation against a key holding the wrong kind of value + exceptionString = ex.what(); + // Command execution error. + exceptionType = REDIS_PLUS_PLUS_REPLY_ERROR; + } catch (const TimeoutError &ex) { + // Reading or writing timeout + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const ClosedError &ex) { + // Connection has been closed. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const IoError &ex) { + // I/O error on the connection. + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_CONNECTION_ERROR; + } catch (const Error &ex) { + // Other errors + exceptionString = ex.what(); + // Connectivity related error. + exceptionType = REDIS_PLUS_PLUS_OTHER_ERROR; + } + + if(exceptionType != REDIS_PLUS_PLUS_NO_ERROR) { + // Problem removing the members from the store zset. + dbError.set(std::string("Inside removeKeys #7, unable to remove multiple members from a given zset."), DPS_BULK_REMOVE_ZREM_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #7, it failed to remove multiple members from a given zset. " << DPS_BULK_REMOVE_ZREM_ERROR, "RedisClusterPlusPlusDBLayer"); + return; + } + } // End of removeKeys method. RedisClusterPlusPlusDBLayerIterator::RedisClusterPlusPlusDBLayerIterator() { diff --git a/com.ibm.streamsx.dps/impl/src/RedisDBLayer.cpp b/com.ibm.streamsx.dps/impl/src/RedisDBLayer.cpp index b507041..b5f9dd9 100644 --- a/com.ibm.streamsx.dps/impl/src/RedisDBLayer.cpp +++ b/com.ibm.streamsx.dps/impl/src/RedisDBLayer.cpp @@ -105,6 +105,7 @@ paragraph. #include #include #include +#include // On the IBM Streams application Linux machines, it is a must to install opensssl and // and openssl-devel RPM packages. The following include files and the SSL custom // initialization logic below in this file will require openssl to be available. @@ -1089,7 +1090,7 @@ namespace distributed } // Apr/16/2022 - // We may have a Redis Sorted Set associated with this store being removed. That set is used for bulk APIs. We can remove it now as the store itself is gone now. + // We will have a Redis Sorted Set associated with this store being removed. That set is used for bulk APIs. We can remove it now as the store itself is gone now. string keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; partitionIdx = getRedisServerPartitionIndex(keyString); cmd = string(REDIS_DEL_CMD) + keyString; @@ -1146,6 +1147,11 @@ namespace distributed return(false); } + // Adding a key will be a two step process by doing both HSET and ZADD one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // string base64_encoded_data_item_key; base64_encode(data_item_key, base64_encoded_data_item_key); @@ -1170,9 +1176,11 @@ namespace distributed freeReplyObject(redis_reply); + // Step 2: + // // Apr/15/2022. // We will also add the key we stored above in a Redis sorted set. That will be useful for - // implementing bulk APIs (donw in later part of this source file) to get a range of keys. + // implementing bulk APIs (done in later part of this source file) to get a range of keys. // This action is performed on the Store Ordered Keys Set that takes the following format. // '101' + 'store id' => 'Redis SortedSet' keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; @@ -1271,6 +1279,11 @@ namespace distributed return(false); } + // Adding a key will be a two step process by doing both HSET and ZADD one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // string base64_encoded_data_item_key; base64_encode(data_item_key, base64_encoded_data_item_key); @@ -1295,6 +1308,8 @@ namespace distributed return(false); } + // Step 2: + // // Apr/15/2022. // We will also add the key we stored above in a Redis sorted set. That will be useful for // implementing bulk APIs (donw in later part of this source file) to get a range of keys. @@ -1690,6 +1705,11 @@ namespace distributed return(false); } + // Removing a key will be a two step process by doing both HDEL and ZREM one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // string base64_encoded_data_item_key; base64_encode(data_item_key, base64_encoded_data_item_key); string cmd = string(REDIS_HDEL_CMD) + keyString + " " + base64_encoded_data_item_key; @@ -1723,6 +1743,8 @@ namespace distributed freeReplyObject(redis_reply); + // Step 2: + // // Apr/16/2022 // I recently added support for bulk APIs. For that, I had to use a sorted set and keep all the // store keys in sorted order. Since we removed a store key above, it is necessary to remove that @@ -2622,7 +2644,7 @@ namespace distributed return(false); } - // We are going to use the RedisCommandArgv to push different parts of the Redis command as passed by the caller. + // We are going to use the redisCommandArgv to push different parts of the Redis command as passed by the caller. vector argv; vector argvlen; @@ -2979,148 +3001,149 @@ namespace distributed /// @param keys User provided mutable list variable. This list must be suitable for storing multiple keys found in a given store and it must be made of a given store's key data type. /// @param keyStartPosition User can indicate a start position from where keys should be fetched and returned. It must be greater than or equal to zero. If not, this API will return back with an empty list of keys. /// @param numberOfKeysNeeded User can indicate the total number of keys to be returned as available from the given key start position. It must be greater than or equal to 0 and less than or equal to 50000. If it is set to 0, then all the available keys upto a maximum of 50000 keys from the given key start position will be returned. - /// @param err Contains the error code. Will be '0' if no error occurs, and a non-zero value otherwise. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get keys operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// - void RedisDBLayer::getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError) { - SPLAPPTRC(L_DEBUG, "Inside getKeys for store id " << store, "RedisDBLayer"); - - std::ostringstream storeId; - storeId << store; - string storeIdString = storeId.str(); - string cmd = ""; - string data_item_key = ""; - - // Before doing anything here, let us validate the user given values for start position and number of keys needed. - if(keyStartPosition < 0) { - dbError.set("A negative value was given for key start position. Error location: Get Keys.", DPS_NEGATIVE_KEY_START_POS_ERROR); - SPLAPPTRC(L_DEBUG, "Inside getKeys, it failed for store " << storeIdString << ". A negative value was given for key start position." << DPS_NEGATIVE_KEY_START_POS_ERROR, "RedisDBLayer"); - return; - } - - // User can provide the numberOfKeysNeeded from 0 to 50000. Validate that. - if(numberOfKeysNeeded < 0 || numberOfKeysNeeded > 50000) { - dbError.set("Invalid value given for number of keys needed. Valid range is from 0 to 50000. Error location: Get Keys.", DPS_INVALID_NUM_KEYS_NEEDED_ERROR); - SPLAPPTRC(L_DEBUG, "Inside getKeys, it failed for store " << storeIdString << ". Invalid value given for number of keys needed. Valid range is from 0 to 50000. " << DPS_INVALID_NUM_KEYS_NEEDED_ERROR, "RedisDBLayer"); - return; - } - - int keyEndPosition = 0; + void RedisDBLayer::getKeys(uint64_t store, std::vector & keysBuffer, std::vector & keysSize, int32_t keyStartPosition, int32_t numberOfKeysNeeded, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getKeys for store id " << store, "RedisDBLayer"); + + std::ostringstream storeId; + storeId << store; + string storeIdString = storeId.str(); + string cmd = ""; + string data_item_key = ""; + + // Before doing anything here, let us validate the user given values for start position and number of keys needed. + if(keyStartPosition < 0) { + dbError.set("Inside getKeys #1, a negative value was given for key start position. Error location: Get Keys.", DPS_NEGATIVE_KEY_START_POS_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getKeys #1, it failed for store " << storeIdString << ". A negative value was given for key start position." << DPS_NEGATIVE_KEY_START_POS_ERROR, "RedisDBLayer"); + return; + } - // If user specified a value of 0 for number of keys needed, we will set it to a - // block of 50000 keys to be returned back. - if(numberOfKeysNeeded == 0) { - // Sorted set is zero based. So, one less than 50000. - keyEndPosition = keyStartPosition + 49999; - } else { - // Sorted set is zero based. So, one less than what the user asked for. - keyEndPosition = keyStartPosition + numberOfKeysNeeded - 1; - } + // User can provide the numberOfKeysNeeded from 0 to 50000. Validate that. + if(numberOfKeysNeeded < 0 || numberOfKeysNeeded > 50000) { + dbError.set("Inside getKeys #2, invalid value given for number of keys needed. Valid range is from 0 to 50000. Error location: Get Keys.", DPS_INVALID_NUM_KEYS_NEEDED_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getKeys #2, it failed for store " << storeIdString << ". Invalid value given for number of keys needed. Valid range is from 0 to 50000. " << DPS_INVALID_NUM_KEYS_NEEDED_ERROR, "RedisDBLayer"); + return; + } - std::ostringstream ksp; - ksp << keyStartPosition; - std::ostringstream kep; - kep << keyEndPosition; - - // In the caller provided list (vector), we will return back the unsigned char * pointer for every key found in the store. Clear that vector now. - keysBuffer.clear(); - // In the caller provided list (vector), we will return back the uint32_t size of every key found in the store. Clear that vector now. - keysSize.clear(); - - // For getting the keys in bulk, we have a Redis Sorted Set for every store that contains - // all the keys preent in the store. It is keeping them in sorted order. That will help us - // to deterministically return the keys to the caller based on a given range. - // We will do it on a best effort basis to achieve good performande by skipping the - // store existence check, store empty check etc. - string keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; - int32_t partitionIdx = getRedisServerPartitionIndex(keyString); + int keyEndPosition = 0; - // Return now if there is no valid connection to the Redis server. - if (redisPartitions[partitionIdx].rdsc == NULL) { - dbError.set("There is no valid connection to the Redis server at this time. Error location: Get Keys.", DPS_CONNECTION_ERROR); - SPLAPPTRC(L_DEBUG, "Inside getKeys, it failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time." << DPS_CONNECTION_ERROR, "RedisDBLayer"); - return; - } + // If user specified a value of 0 for number of keys needed, we will set it to a + // block of 50000 keys to be returned back. + if(numberOfKeysNeeded == 0) { + // Sorted set is zero based. So, one less than 50000. + keyEndPosition = keyStartPosition + 49999; + } else { + // Sorted set is zero based. So, one less than what the user asked for. + keyEndPosition = keyStartPosition + numberOfKeysNeeded - 1; + } - // Let us get the available keys from this store's sorted set. - cmd = string(REDIS_ZRANGE_CMD) + keyString + string(" ") + - ksp.str() + string(" ") + kep.str(); - redis_reply = (redisReply*)redisCommand(redisPartitions[partitionIdx].rdsc, cmd.c_str()); + std::ostringstream ksp; + ksp << keyStartPosition; + std::ostringstream kep; + kep << keyEndPosition; + + // In the caller provided list (vector), we will return back the unsigned char * pointer for every key found in the store. Clear that vector now. + keysBuffer.clear(); + // In the caller provided list (vector), we will return back the uint32_t size of every key found in the store. Clear that vector now. + keysSize.clear(); + + // For getting the keys in bulk, we have a Redis Sorted Set for every store that contains + // all the keys preent in the store. It is keeping them in sorted order. That will help us + // to deterministically return the keys to the caller based on a given range. + // We will do it on a best effort basis to achieve good performande by skipping the + // store existence check, store empty check etc. + string keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; + int32_t partitionIdx = getRedisServerPartitionIndex(keyString); - if (redis_reply == NULL) { - dbError.set("Unable to connect to the redis server(s). Error location: ZRANGE command. " + std::string(redisPartitions[partitionIdx].rdsc->errstr), DPS_CONNECTION_ERROR); - return; - } + // Return now if there is no valid connection to the Redis server. + if (redisPartitions[partitionIdx].rdsc == NULL) { + dbError.set("Inside getKeys #3, there is no valid connection to the Redis server at this time. Error location: Get Keys.", DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getKeys #3, it failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time." << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } - if (redis_reply->type == REDIS_REPLY_ERROR) { - // Unable to get keys for the store. - dbError.set("Unable to get bulk keys via ZRANGE for the StoreId " + storeIdString + - ". " + std::string(redis_reply->str), DPS_GET_STORE_DATA_ITEM_KEYS_ERROR); - SPLAPPTRC(L_DEBUG, "Inside getKeys, ZRANGE failed for store id " << storeIdString << ". " << DPS_GET_STORE_DATA_ITEM_KEYS_ERROR, "RedisDBLayer"); - freeReplyObject(redis_reply); - return; - } + // Let us get the available keys from this store's sorted set. + cmd = string(REDIS_ZRANGE_CMD) + keyString + string(" ") + ksp.str() + string(" ") + kep.str(); + redis_reply = (redisReply*)redisCommand(redisPartitions[partitionIdx].rdsc, cmd.c_str()); - if (redis_reply->type != REDIS_REPLY_ARRAY) { - // Unable to get data item keys from the store in an array format. - dbError.set("Unable to get bulk keys via ZRANGE in an array format for the StoreId " + storeIdString + - ". " + std::string(redis_reply->str), DPS_GET_STORE_DATA_ITEM_KEYS_AS_AN_ARRAY_ERROR); - SPLAPPTRC(L_DEBUG, "Inside getKeys, ZRANGE failed for store id " << storeIdString << ". " << DPS_GET_STORE_DATA_ITEM_KEYS_AS_AN_ARRAY_ERROR, "RedisDBLayer"); - freeReplyObject(this->redis_reply); - return; - } + if (redis_reply == NULL) { + dbError.set("Inside getKeys #4, unable to connect to the redis server(s). Error location: ZRANGE command. " + std::string(redisPartitions[partitionIdx].rdsc->errstr), DPS_CONNECTION_ERROR); + return; + } - // We have the data item keys returned in array now. - // Let us insert them into the caller provided list (vector) that will hold the data item keys for this store. - for (unsigned int j = 0; j < redis_reply->elements; j++) { - data_item_key = string(redis_reply->element[j]->str); + if (redis_reply->type == REDIS_REPLY_ERROR) { + // Unable to get keys for the store. + dbError.set("Inside getKeys #5, unable to get bulk keys via ZRANGE for the StoreId " + storeIdString + ". " + std::string(redis_reply->str), DPS_GET_STORE_DATA_ITEM_KEYS_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getKey #5, ZRANGE failed for store id " << storeIdString << ". " << DPS_GET_STORE_DATA_ITEM_KEYS_ERROR, "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } - // In order to support spaces in data item keys, we base64 encoded them before storing it in Redis. - // Let us base64 decode it now to get the original data item key. - string base64_decoded_data_item_key; - base64_decode(data_item_key, base64_decoded_data_item_key); - data_item_key = base64_decoded_data_item_key; - uint32_t keySize = data_item_key.length(); - // Allocate memory for this key and copy it to that buffer. - unsigned char * keyData = (unsigned char *) malloc(keySize); + if (redis_reply->type != REDIS_REPLY_ARRAY) { + // Unable to get data item keys from the store in an array format. + dbError.set("Inside getKeys #6, unable to get bulk keys via ZRANGE in an array format for the StoreId " + storeIdString + ". " + std::string(redis_reply->str), DPS_GET_STORE_DATA_ITEM_KEYS_AS_AN_ARRAY_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getKeys #6, ZRANGE failed for store id " << storeIdString << ". " << DPS_GET_STORE_DATA_ITEM_KEYS_AS_AN_ARRAY_ERROR, "RedisDBLayer"); + freeReplyObject(this->redis_reply); + return; + } - if (keyData == NULL) { - // This error will occur very rarely. - // If it happens, we will handle it. + // We have the data item keys returned in array now. + // Let us insert them into the caller provided list (vector) that will hold the data item keys for this store. + for (unsigned int j = 0; j < redis_reply->elements; j++) { + data_item_key = string(redis_reply->element[j]->str); + + // In order to support spaces in data item keys, we base64 encoded them before storing it in Redis. + // Let us base64 decode it now to get the original data item key. + string base64_decoded_data_item_key; + base64_decode(data_item_key, base64_decoded_data_item_key); + data_item_key = base64_decoded_data_item_key; + uint32_t keySize = data_item_key.length(); + // Allocate memory for this key and copy it to that buffer. + unsigned char * keyData = (unsigned char *) malloc(keySize); + + if (keyData == NULL) { + // This error will occur very rarely. + // If it happens, we will handle it. freeReplyObject(redis_reply); // We will leave any partially filled data in the user provided list. // On detection the db error flag, caller should take the necessary steps to // free the allocated memory for the partially filled data in that list. - dbError.set("Unable to allocate memory for the keyData while fetching bulk keys in getKeys for the StoreId " + - storeIdString + ".", DPS_STORE_ITERATION_MALLOC_ERROR); - SPLAPPTRC(L_DEBUG, "Inside getKeys, memory allocation failed for store id " << storeIdString << ". " << DPS_STORE_ITERATION_MALLOC_ERROR, "RedisDBLayerr"); - return; - } + dbError.set("Inside getKeys #7, unable to allocate memory for the keyData while fetching bulk keys in getKeys for the StoreId " + storeIdString + ".", DPS_STORE_ITERATION_MALLOC_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getKeys #7, memory allocation failed for store id " << storeIdString << ". " << DPS_STORE_ITERATION_MALLOC_ERROR, "RedisDBLayer"); + return; + } - // Copy the raw key data into the allocated buffer. - memcpy(keyData, data_item_key.data(), keySize); - // We are done. We expect the caller to free the keyData and valueData buffers. + // Copy the raw key data into the allocated buffer. + memcpy(keyData, data_item_key.data(), keySize); + // We are done. We expect the caller to free the keyData and valueData buffers. // Let us append the key and the key size to the user provided lists. - keysBuffer.push_back(keyData); + keysBuffer.push_back(keyData); keysSize.push_back(keySize); - } // End of for loop. + } // End of for loop. - freeReplyObject(redis_reply); + freeReplyObject(redis_reply); } // End of getKeys method. // Senthil added this on Apr/18/2022. - // This method will get the value for a given key from the given store without - // performing any checks for the existence of store, key etc. This is a slightly + // This method will get the values for a given list of keys from the given store without + // performing any checks for the existence of store, key etc. This is a // faster version of the get method above. /// /// @param store The handle of the store given in a string form. /// @param key User provided key for which a value needs to be fetched. - /// @param keySixe Size of the buffer holding the key. + /// @param keySize Size of the buffer holding the key. /// @param value Buffer pointer where the fetched value will be made available for the caller. /// @param valueSize Size of the buffer holding the value. - /// @param error Error code from the get operation if any will be made available for the caller. + /// @param err User provided mutable uint64 typed variable in which the result of this bulk get values operation will be returned. It will be 0 if no error occurs and a non-zero error code otherwise. /// - void RedisDBLayer::getValue(std::string const & storeIdString, char const * & key, uint32_t const & keySize, unsigned char * & value, uint32_t & valueSize, uint64_t & error) { - SPLAPPTRC(L_DEBUG, "Inside getValue for store id " << storeIdString, "RedisDBLayer"); + void RedisDBLayer::getValues(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & valueData, std::vector & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside getValues for store id " << store, "RedisDBLayer"); + + // Create a string form of the store id. + std::ostringstream storeId; + storeId << store; + std::string storeIdString = storeId.str(); // This method will skip all the safety checks such as store existence, key existence etc. // If store safety is needed, the caller of this API can do their own DPS locks at the application level. @@ -3128,75 +3151,578 @@ namespace distributed string keyString = string(DPS_STORE_CONTENTS_HASH_TYPE) + storeIdString; int32_t partitionIdx = getRedisServerPartitionIndex(keyString); - value = NULL; - valueSize = 0; // Return now if there is no valid connection to the Redis server. if (redisPartitions[partitionIdx].rdsc == NULL) { - error = DPS_CONNECTION_ERROR; - SPLAPPTRC(L_DEBUG, "Inside getValue #1: It failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time. " << DPS_CONNECTION_ERROR, "RedisDBLayer"); + dbError.set(std::string("Inside getValues #1, Tthere is no valid connection to the Redis server at this time."), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getValues #1, it failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time. " << DPS_CONNECTION_ERROR, "RedisDBLayer"); return; } - // In our Redis dps implementation, data item keys can have space characters. - // Original NBF serialized key is stored in Redis as a base64 encoded string. - // So, to query, we will have to pass a base64 encoded key. - string base64_encoded_data_item_key; - base64_encode(string(key, keySize), base64_encoded_data_item_key); + // Redis HMGET command is what we want to read multiple values in one API call. + string argvStyleRedisCommand = string(REDIS_HMGET_CMD); + // Strip the space at the end of the command that should not be there for the argv style Redis command. + argvStyleRedisCommand = argvStyleRedisCommand.substr(0, argvStyleRedisCommand.size()-1); - // Fetch the data item now. - string cmd = string(REDIS_HGET_CMD) + keyString + " " + base64_encoded_data_item_key; - redis_reply = (redisReply*)redisCommand(redisPartitions[partitionIdx].rdsc, cmd.c_str()); + // We will use the Redis variadic API to do multiple gets in one call to that API. + vector argv; + vector argvlen; + + // Let us push the Redis command string and its size. + argv.push_back(argvStyleRedisCommand.c_str()); + argvlen.push_back(argvStyleRedisCommand.length()); + + // Let us push the keystring now. + argv.push_back(keyString.c_str()); + argvlen.push_back(keyString.length()); + + // We should have all the base64 encoded keys in their own memory locations instead of + // keep reusing a single local variable. + std::vector base64_keys; + + // Initialize the base64 keys vector with empty string elements. + for(int i=0; i<(int)keyData.size(); i++) { + base64_keys.push_back(std::string("")); + } + + // We are given all the keys in a separate list. + // That list is holding the NBF formatted buffers for the keys. + // We can iterate over that list and keep populating two other lists required for + // executing the variadic version of the Redis command API. For binary safe data, + // we have to populate the size of every data item in the argvlen vector. + for(int i=0; i<(int)keyData.size(); i++) { + // Base64 encode a key. + base64_encode(string(keyData.at(i), keySize.at(i)), base64_keys.at(i)); + // Push a key now. + argv.push_back(base64_keys.at(i).c_str()); + // Push the size of the key now. + argvlen.push_back(base64_keys.at(i).length()); + } + + // Execute the variadic Redis Argv command. + redis_reply = (redisReply*) redisCommandArgv(redisPartitions[partitionIdx].rdsc, argv.size(), &(argv[0]), &(argvlen[0])); if (redis_reply == NULL) { - error = DPS_REDIS_REPLY_NULL_ERROR; - SPLAPPTRC(L_DEBUG, "Inside getValue #2: Unable to connect to the redis server(s). " << std::string(redisPartitions[partitionIdx].rdsc->errstr), "RedisDBLayer"); + dbError.set(std::string("Inside getValues #2, unable to connect to the redis server(s). ") + std::string(redisPartitions[partitionIdx].rdsc->errstr), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getValues #2, it failed for executing the HMGET command for multiple keys. Error=" << std::string(redisPartitions[partitionIdx].rdsc->errstr) << ". " << DPS_CONNECTION_ERROR, "RedisDBLayer"); return; } - // If SUCCESS, this result can come as an empty string. if (redis_reply->type == REDIS_REPLY_ERROR) { - error = DPS_DATA_ITEM_READ_ERROR; - SPLAPPTRC(L_DEBUG, "Inside getValue #3: Unable to get the requested data item from the store with the StoreId " << storeIdString + ". " + std::string(redis_reply->str), "RedisDBLayer"); + // Problem in reading data items from the cache. + dbError.set(std::string("Inside getValues #3, unable to get data items via HMGET. ") + std::string(redis_reply->str), DPS_BULK_GET_HMGET_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getValues #3, it failed to get data items via HMGET. Error=" << std::string(redis_reply->str) << ". " << DPS_BULK_GET_HMGET_ERROR, "RedisDBLayer"); freeReplyObject(redis_reply); return; } - if (redis_reply->type == REDIS_REPLY_NIL) { - error = DPS_REDIS_REPLY_NIL_ERROR; - // Unable to get the requested data item from the cache. - SPLAPPTRC(L_DEBUG, "Inside getValue #4: The requested data item doesn't exist in the StoreId " << storeIdString, "RedisDBLayer"); + // We expect a Redis reply array as a result type in this case. + if (redis_reply->type != REDIS_REPLY_ARRAY) { + // Problem in getting multi value result as an array result type.. + dbError.set(std::string("Inside getValues #4, unable to get data items results in an array. ") + std::string(redis_reply->str), DPS_BULK_GET_HMGET_NO_REPLY_ARRAY_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getValues #4, it failed to get data items results in an array. Error=" << std::string(redis_reply->str) << ". " << DPS_BULK_GET_HMGET_NO_REPLY_ARRAY_ERROR, "RedisDBLayer"); freeReplyObject(redis_reply); return; } - // Data item value read from the store will be in this format: 'value' - if ((unsigned)redis_reply->len > 0) { - // We can allocate memory for the exact length of the data item value. - valueSize = redis_reply->len; - value = (unsigned char *) malloc(valueSize); - - if (value == NULL) { - valueSize = 0; - error = DPS_GET_DATA_ITEM_MALLOC_ERROR; - // Unable to allocate memory to transfer the data item value. - SPLAPPTRC(L_DEBUG, "Inside getValue #5: Unable to allocate memory to copy the data item value for the StoreId " << storeIdString << ".", "RedisDBLayer"); + // We have the data item keys returned in an array now. + // Let us start collecting them as char array pointers to be sent back to the caller. + for (unsigned int j = 0; j < redis_reply->elements; j++) { + if(redis_reply->element[j]->str == NULL) { + SPLAPPTRC(L_DEBUG, "Inside getValues #5, it returned a NULL result for key index " << j, "RedisDBLayer"); + valueData.push_back(NULL); + valueSize.push_back(0); + continue; + } + + uint32_t myValueSize = redis_reply->element[j]->len; + + // Data item value read from the store will be in this format: 'value' + if (myValueSize > 0) { + // We can allocate memory for the exact length of the data item value. + unsigned char * value = (unsigned char *) malloc(myValueSize); + + if (value == NULL) { + dbError.set(std::string("Inside getValues #6, unable to allocate memory to copy the data item value for the storeId ") + storeIdString, DPS_BULK_GET_HMGET_MALLOC_ERROR); + // Unable to allocate memory to transfer the data item value. + SPLAPPTRC(L_DEBUG, "Inside getValues #6: Unable to allocate memory to copy the data item value for the StoreId " << storeIdString << ".", "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } else { + // We expect the caller of this method to free the value pointer. + memcpy(value, redis_reply->element[j]->str, myValueSize); + // We can add it to the user provided lists. + valueData.push_back(value); + valueSize.push_back(myValueSize); + } } else { - // We expect the caller of this method to free the value pointer. - memcpy(value, redis_reply->str, valueSize); - error = 0; + // We got a zero length data item value. + // This is not very useful and it should never happen. + // We will raise an error for this. + dbError.set(std::string("Inside getValues #7, Found an empty data item value for the storeId ") + storeIdString, DPS_BULK_GET_HMGET_EMPTY_VALUE_ERROR); + SPLAPPTRC(L_DEBUG, "Inside getValues #7: Found an empty data item value for the StoreId " << storeIdString << ".", "RedisDBLayer"); + freeReplyObject(redis_reply); + return; } - } else { - // We got a zero length data item value. - // This is not very useful and it should never happen. - // We will raise an error for this. - valueSize = 0; - error = DPS_EMPTY_DATA_ITEM_VALUE_FOUND_ERROR; - SPLAPPTRC(L_DEBUG, "Inside getValue #6: Found an empty data item value for the StoreId " << storeIdString << ".", "RedisDBLayer"); - } // End of if ((unsigned)redis_reply->len > 0) + } // End of for loop. + + freeReplyObject(redis_reply); + } // End of getValues method. + + // Senthil added this on Apr/21/2022. + // This method will put the given K/V pairs in a given store. + // It is a bulk put operation. + void RedisDBLayer::putKVPairs(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector const & valueData, std::vector const & valueSize, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside putKVPairs for store id " << store, "RedisDBLayer"); + + // Get the string form of the store id. + std::ostringstream storeId; + storeId << store; + string storeIdString = storeId.str(); + + // Adding keys will be a two step process by doing both HSET and ZADD one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // + string keyString = string(DPS_STORE_CONTENTS_HASH_TYPE) + storeIdString; + int32_t partitionIdx = getRedisServerPartitionIndex(keyString); + + // Return now if there is no valid connection to the Redis server. + if (redisPartitions[partitionIdx].rdsc == NULL) { + dbError.set(std::string("Inside putKVPairs #1, there is no valid connection to the Redis server at this time."), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside putKVPairs #1, it failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time. " << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + // Redis HSET command now supports storing multiple K/V pairs. + string argvStyleRedisCommand = string(REDIS_HSET_CMD); + // Strip the space at the end of the command that should not be there for the argv style Redis command. + argvStyleRedisCommand = argvStyleRedisCommand.substr(0, argvStyleRedisCommand.size()-1); + + // We will use the Redis variadic API to do multiple puts in one call to that API. + vector argv; + vector argvlen; + + // Let us push the Redis command string and its size. + argv.push_back(argvStyleRedisCommand.c_str()); + argvlen.push_back(argvStyleRedisCommand.length()); + + // Let us push the keystring now. + argv.push_back(keyString.c_str()); + argvlen.push_back(keyString.length()); + + // We should have all the base64 encoded keys in their own memory locations instead of + // keep reusing a single local variable. + std::vector base64_keys; + + // Initialize the base64 keys vector with empty string elements. + for(int i=0; i<(int)keyData.size(); i++) { + base64_keys.push_back(std::string("")); + } + + // We are given all the keys and the corresponding values in two separate lists. + // Those two lists are holding the NBF formatted buffers for the keys and values. + // We can iterate over the lists and keep populating two other lists required for + // executing the variadic version of the Redis command API. For binary safe data, + // we have to populate the size of every data item in the argvlen vector. + for(int i=0; i<(int)keyData.size(); i++) { + // Base64 encode a key. + base64_encode(string(keyData.at(i), keySize.at(i)), base64_keys.at(i)); + // Push a key now. + argv.push_back(base64_keys.at(i).c_str()); + // Push the size of the key now. + argvlen.push_back(base64_keys.at(i).length()); + // Push a value now. + argv.push_back((const char*)(valueData.at(i))); + // Push the size of the value now. + argvlen.push_back(valueSize.at(i)); + } + + // Execute the variadic Redis Argv command. + redis_reply = (redisReply*) redisCommandArgv(redisPartitions[partitionIdx].rdsc, argv.size(), &(argv[0]), &(argvlen[0])); + + if (redis_reply == NULL) { + dbError.set(std::string("Inside putKVPairs #2, unable to connect to the redis server(s). ") + std::string(redisPartitions[partitionIdx].rdsc->errstr), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside putKVPairs #2, it failed for executing the hset command for multiple K/V pairs. Error=" << std::string(redisPartitions[partitionIdx].rdsc->errstr) << ". " << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + if (redis_reply->type == REDIS_REPLY_ERROR) { + // Problem in storing a data item in the cache. + dbError.set(std::string("Inside putKVPairs #3, unable to store a data item. ") + std::string(redis_reply->str), DPS_BULK_PUT_HSET_ERROR); + SPLAPPTRC(L_DEBUG, "Inside putKVPairs #3, it failed to store a data item. Error=" << std::string(redis_reply->str) << ". " << DPS_BULK_PUT_HSET_ERROR, "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } + + freeReplyObject(redis_reply); + + // Step 2: + // + // Now that we have put multiple K/V Pairs, let us now add the keys for them in + // zset (Sorted Set) that we keep for every DPS store. That will be useful for + // implementing bulk APIs to get a range of keys. + // This action is performed on the Store Ordered Keys Set that takes the following format. + // '101' + 'store id' => 'Redis SortedSet' + keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; + partitionIdx = getRedisServerPartitionIndex(keyString); + + // Return now if there is no valid connection to the Redis server. + if (redisPartitions[partitionIdx].rdsc == NULL) { + dbError.set("Inside putKVPairs #4, there is no valid connection to the Redis server at this time. Error location: putKVPairs-->Store key in SortedSet.", DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside putKVPairs #4, it failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time. Error location: Store key in SortedSet" << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + argvStyleRedisCommand = string(REDIS_ZADD_CMD); + // Strip the space at the end of the command that should not be there for the argv style Redis command. + argvStyleRedisCommand = argvStyleRedisCommand.substr(0, argvStyleRedisCommand.size()-1); + + // Clear the vectors. + argv.clear(); + argvlen.clear(); + + // Let us push the Redis command string and its size. + argv.push_back(argvStyleRedisCommand.c_str()); + argvlen.push_back(argvStyleRedisCommand.length()); + + // Let us push the keystring now. + argv.push_back(keyString.c_str()); + argvlen.push_back(keyString.length()); + + std::string nxString = "NX"; + // Add the NX flag to add only if "Not Exists". + argv.push_back(nxString.c_str()); + argvlen.push_back(nxString.length()); + + // We should have all the zset scores in their own memory locations instead of + // keep reusing a single local variable. + std::vector zset_scores; + + // Stay in a loop and add all the zset scores and members now. + for(int i=0; i<(int)keyData.size(); i++) { + // Generate a random number between 0 and 1. + zset_scores.push_back(streams_boost::to_string(SPL::Functions::Math::random())); + // Push a score now + argv.push_back(zset_scores.at(i).c_str()); + // Push the size of the score now. + argvlen.push_back(zset_scores.at(i).length()); + // Push a member now. + argv.push_back(base64_keys.at(i).c_str()); + // Push the size of the member now. + argvlen.push_back(base64_keys.at(i).length()); + } + + // Execute the variadic Redis Argv command. + redis_reply = (redisReply*) redisCommandArgv(redisPartitions[partitionIdx].rdsc, argv.size(), &(argv[0]), &(argvlen[0])); + + if (redis_reply == NULL) { + dbError.set(std::string("Inside putKVPairs #5, unable to connect to the redis server(s). ") + std::string(redisPartitions[partitionIdx].rdsc->errstr), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside putKVPairs #5, it failed for executing the zadd command for multiple keys. Error=" << std::string(redisPartitions[partitionIdx].rdsc->errstr) << ". " << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + if (redis_reply->type == REDIS_REPLY_ERROR) { + // Problem in storing a key in zset + dbError.set(std::string("Inside putKVPairs #6, unable to store a key in zset. ") + std::string(redis_reply->str), DPS_BULK_PUT_ZADD_ERROR); + SPLAPPTRC(L_DEBUG, "Inside putKVPairs #6, it failed to store a key in zset. Error=" << std::string(redis_reply->str) << ". " << DPS_BULK_PUT_ZADD_ERROR, "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } + + freeReplyObject(redis_reply); + } // End of putKVPairs method. + + // Senthil added this on Apr/27/2022. + // This method checks for the existence of a given list of keys in a given store and returns the true or false results. + // This is a bulk key existence check operation. + void RedisDBLayer::hasKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, std::vector & results, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys for store id " << store, "RedisDBLayer"); + + // Get the string form of the store id. + std::ostringstream storeId; + storeId << store; + string storeIdString = storeId.str(); + + // To check the existence of multiple keys in a store via a single API call, + // Redis doesn't have a straightforward command that can be executed through + // the variadic API redisCommandArgv. In fact, until the Redis v6.2.0 release, + // it was impossible to do. In v6.2.0, they introduced ZMSCORE to check multiple + // members' scores in a sorted set. Since we recently introduced in DPS to + // shadow every store hash with a zset to support some of the bulk APIs, we are + // going to rely on the new ZMSCORE redis command to check existence of multiple keys. + // Because of the use of ZMSCORE, DPS toolkit will require Redis v6.2.0 or a + // higher version going forward from this date (Apr/27/2022). + // + string keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; + int32_t partitionIdx = getRedisServerPartitionIndex(keyString); + + // Return now if there is no valid connection to the Redis server. + if (redisPartitions[partitionIdx].rdsc == NULL) { + dbError.set("Inside hasKeys #1, there is no valid connection to the Redis server at this time. Error location: hasKeys-->Prepare to do ZMSCORE in the store's SortedSet.", DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #1, it failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time. Error location: hasKeys-->Prepare to do ZMSCORE in the store's SortedSet." << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + // Prepare to run the Redis ZMSCORE command via redis argv style command. + string argvStyleRedisCommand = string(REDIS_ZMSCORE_CMD); + // Strip the space at the end of the command that should not be there for the argv style Redis command. + argvStyleRedisCommand = argvStyleRedisCommand.substr(0, argvStyleRedisCommand.size()-1); + + // We will use the Redis variadic API to do multiple puts in one call to that API. + vector argv; + vector argvlen; + + // Let us push the Redis command string and its size. + argv.push_back(argvStyleRedisCommand.c_str()); + argvlen.push_back(argvStyleRedisCommand.length()); + + // Let us push the keystring now. + argv.push_back(keyString.c_str()); + argvlen.push_back(keyString.length()); + + // We should have all the base64 encoded keys in their own memory locations instead of + // keep reusing a single local variable. + std::vector base64_keys; + + // Initialize the base64 keys vector with empty string elements. + for(int i=0; i<(int)keyData.size(); i++) { + base64_keys.push_back(std::string("")); + } + + // We are given all the keys in a separate list. + // That list is holding the NBF formatted buffers for the keys. + // We can iterate over that list and keep populating another list required for + // executing the variadic version of the Redis command API. For binary safe data, + // we have to populate the size of every item in the argvlen vector. + for(int i=0; i<(int)keyData.size(); i++) { + // Base64 encode a key. + base64_encode(string(keyData.at(i), keySize.at(i)), base64_keys.at(i)); + // Push a key now. + argv.push_back(base64_keys.at(i).c_str()); + // Push the size of the key now. + argvlen.push_back(base64_keys.at(i).length()); + } + + // Execute the variadic Redis Argv command. + redis_reply = (redisReply*) redisCommandArgv(redisPartitions[partitionIdx].rdsc, argv.size(), &(argv[0]), &(argvlen[0])); + + if (redis_reply == NULL) { + dbError.set(std::string("Inside hasKeys #2, unable to connect to the redis server(s). ") + std::string(redisPartitions[partitionIdx].rdsc->errstr), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #2, It failed for executing the ZMSCORE command for multiple keys. Error=" << std::string(redisPartitions[partitionIdx].rdsc->errstr) << ". " << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + if (redis_reply->type == REDIS_REPLY_ERROR) { + // Problem reading scores for the zset members. + dbError.set(std::string("Inside hasKeys #3, unable to get scores for the given keys. ") + std::string(redis_reply->str), DPS_BULK_GET_ZMSCORE_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #3, it failed to get scores for the given keys. Error=" << std::string(redis_reply->str) << ". " << DPS_BULK_GET_ZMSCORE_ERROR, "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } + + // We expect a Redis reply array as a result type in this case. + if (redis_reply->type != REDIS_REPLY_ARRAY) { + // Problem in getting multi value result as an array result type.. + dbError.set(std::string("Inside hasKeys #4, unable to get the scores in an array. ") + std::string(redis_reply->str), DPS_BULK_GET_ZMSCORE_NO_REPLY_ARRAY_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #4, it failed to get the scores in an array. Error=" << std::string(redis_reply->str) << ". " << DPS_BULK_GET_ZMSCORE_NO_REPLY_ARRAY_ERROR, "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } + + // We have the scores for the given keys returned in a Redis array now. + // Non-existing keys will result in Redis Nil. + // Let us start collecting them as boolean true or false to be sent back to the caller. + for (unsigned int j = 0; j < redis_reply->elements; j++) { + if(redis_reply->element[j]->str == NULL) { + SPLAPPTRC(L_DEBUG, "Inside hasKeys #5, it returned a NULL result for key index " << j, "RedisDBLayer"); + // This means the given key doesn't exist in the sorted set i.e. in the DPS store hash as well since the + // sorted set mirrors our store hash at all times. + results.push_back(false); + continue; + } + + uint32_t myValueSize = redis_reply->element[j]->len; + + // Score read from the sorted set will be in this format: 'value' + if (myValueSize > 0) { + // This is a valid score. That means the given key exists. + results.push_back(true); + } else { + // We got a zero length score. + // This is not very useful and it should never happen. + // We will raise an error for this. + dbError.set(std::string("Inside hasKeys #6, Found an empty score in zset for the storeId ") + storeIdString, DPS_BULK_GET_ZMSCORE_EMPTY_VALUE_ERROR); + SPLAPPTRC(L_DEBUG, "Inside hasKeys #6: Found an empty score in zset for the StoreId " << storeIdString << ".", "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } + } // End of for loop. + + freeReplyObject(redis_reply); + } // End of hasKeys method. + + // Senthil added this on Apr/28/2022. + // This method removes a given list of keys from a given store. + // This is a bulk key removal operation. + void RedisDBLayer::removeKeys(uint64_t store, std::vector const & keyData, std::vector const & keySize, int32_t & totalKeysRemoved, PersistenceError & dbError) { + SPLAPPTRC(L_DEBUG, "Inside removeKeys for store id " << store, "RedisDBLayer"); + + // Get the string form of the store id. + std::ostringstream storeId; + storeId << store; + string storeIdString = storeId.str(); + + string keyString = string(DPS_STORE_CONTENTS_HASH_TYPE) + storeIdString; + int32_t partitionIdx = getRedisServerPartitionIndex(keyString); + + // Return now if there is no valid connection to the Redis server. + if (redisPartitions[partitionIdx].rdsc == NULL) { + dbError.set(std::string("Inside removeKeys #1, there is no valid connection to the Redis server at this time."), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #1, it failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time. " << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + // Removing keys will be a two step process by doing both HDEL and ZREM one after the other. + // This is needed because we have every DPS store hash being shadowed by a zset to aid in a few bulk operations. + + // Step 1: + // + // Redis HDEL command supports removing multiple keys. + string argvStyleRedisCommand = string(REDIS_HDEL_CMD); + // Strip the space at the end of the command that should not be there for the argv style Redis command. + argvStyleRedisCommand = argvStyleRedisCommand.substr(0, argvStyleRedisCommand.size()-1); + + // We will use the Redis variadic API to do multiple key removals from the store in one call to that API. + vector argv; + vector argvlen; + + // Let us push the Redis command string and its size. + argv.push_back(argvStyleRedisCommand.c_str()); + argvlen.push_back(argvStyleRedisCommand.length()); + + // Let us push the keystring now. + argv.push_back(keyString.c_str()); + argvlen.push_back(keyString.length()); + + // We should have all the base64 encoded keys in their own memory locations instead of + // keep reusing a single local variable. + std::vector base64_keys; + + // Initialize the base64 keys vector with empty string elements. + for(int i=0; i<(int)keyData.size(); i++) { + base64_keys.push_back(std::string("")); + } + + // We are given all the keys in a separate list. + // That list is holding the NBF formatted buffers for the keys. + // We can iterate over that list and keep populating another list required for + // executing the variadic version of the Redis command API. For binary safe data, + // we have to populate the size of every item in the argvlen vector. + for(int i=0; i<(int)keyData.size(); i++) { + // Base64 encode a key. + base64_encode(string(keyData.at(i), keySize.at(i)), base64_keys.at(i)); + // Push a key now. + argv.push_back(base64_keys.at(i).c_str()); + // Push the size of the key now. + argvlen.push_back(base64_keys.at(i).length()); + } + + // Execute the variadic Redis Argv command. + redis_reply = (redisReply*) redisCommandArgv(redisPartitions[partitionIdx].rdsc, argv.size(), &(argv[0]), &(argvlen[0])); + + if (redis_reply == NULL) { + dbError.set(std::string("Inside removeKeys #2, unable to connect to the redis server(s). ") + std::string(redisPartitions[partitionIdx].rdsc->errstr), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #2, it failed for executing the HDEL command for multiple keys. Error=" << std::string(redisPartitions[partitionIdx].rdsc->errstr) << ". " << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + if (redis_reply->type == REDIS_REPLY_ERROR) { + // Problem in removing keys from the cache. + dbError.set(std::string("Inside removeKeys #3, unable to remove keys from a store. ") + std::string(redis_reply->str), DPS_BULK_REMOVE_HDEL_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #3, it failed to remove keys from a store. Error=" << std::string(redis_reply->str) << ". " << DPS_BULK_REMOVE_HDEL_ERROR, "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } + + // We can now get the total number of keys that got removed. + totalKeysRemoved = redis_reply->integer; + freeReplyObject(redis_reply); + + // Step 2: + // + // Now that we have removed multiple keys from the store, let us now remove their + // corresponding members that we keep in a zset (Sorted Set) for every DPS store. + // This action is performed on the Store Ordered Keys Set that takes the following format. + // '101' + 'store id' => 'Redis SortedSet' + keyString = string(DPS_STORE_ORDERED_KEYS_SET_TYPE ) + storeIdString; + partitionIdx = getRedisServerPartitionIndex(keyString); + + // Return now if there is no valid connection to the Redis server. + if (redisPartitions[partitionIdx].rdsc == NULL) { + dbError.set("Inside removeKeys #4, there is no valid connection to the Redis server at this time. Error location: removeKeys-->Remove members in SortedSet.", DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #4, it failed for store " << storeIdString << ". There is no valid connection to the Redis server at this time. Error location: Remove members from SortedSet" << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + argvStyleRedisCommand = string(REDIS_ZREM_CMD); + // Strip the space at the end of the command that should not be there for the argv style Redis command. + argvStyleRedisCommand = argvStyleRedisCommand.substr(0, argvStyleRedisCommand.size()-1); + + // Clear the vectors. + argv.clear(); + argvlen.clear(); + + // Let us push the Redis command string and its size. + argv.push_back(argvStyleRedisCommand.c_str()); + argvlen.push_back(argvStyleRedisCommand.length()); + + // Let us push the keystring now. + argv.push_back(keyString.c_str()); + argvlen.push_back(keyString.length()); + + // Stay in a loop and add all the zset members now. + for(int i=0; i<(int)keyData.size(); i++) { + // Push a member now. + argv.push_back(base64_keys.at(i).c_str()); + // Push the size of the member now. + argvlen.push_back(base64_keys.at(i).length()); + } + + // Execute the variadic Redis Argv command. + redis_reply = (redisReply*) redisCommandArgv(redisPartitions[partitionIdx].rdsc, argv.size(), &(argv[0]), &(argvlen[0])); + + if (redis_reply == NULL) { + dbError.set(std::string("Inside removeKeys #5, unable to connect to the redis server(s). ") + std::string(redisPartitions[partitionIdx].rdsc->errstr), DPS_CONNECTION_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #5, it failed for executing the ZREM command for multiple keys. Error=" << std::string(redisPartitions[partitionIdx].rdsc->errstr) << ". " << DPS_CONNECTION_ERROR, "RedisDBLayer"); + return; + } + + if (redis_reply->type == REDIS_REPLY_ERROR) { + // Problem in removing members from the zset. + dbError.set(std::string("Inside removeKeys #6, unable to remove members from the zset. ") + std::string(redis_reply->str), DPS_BULK_REMOVE_ZREM_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #6, it failed to remove member from the zset. Error=" << std::string(redis_reply->str) << ". " << DPS_BULK_REMOVE_ZREM_ERROR, "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } + + // Get the total number of members removed from the zset. + int32_t total_zrem_cnt = redis_reply->integer; + + // We have to check that we removed the same number of keys in both the + // store hash and store zset. If they are not the same number, it is a serious error that manifested + // either in this method or elsewhere and it will cause key, member imbalance and problems later. + if(totalKeysRemoved != total_zrem_cnt) { + // Remove cnt mismatch between the store and the zset. + dbError.set(std::string("Inside removeKeys #7, key, member removal cnt mismatch between HDEL and ZREM."), DPS_BULK_REMOVE_CNT_MISMATCH_ERROR); + SPLAPPTRC(L_DEBUG, "Inside removeKeys #7, it failed with a key, member removal cnt mismatch between HDEL and ZREM. " << DPS_BULK_REMOVE_CNT_MISMATCH_ERROR, "RedisDBLayer"); + freeReplyObject(redis_reply); + return; + } freeReplyObject(redis_reply); - } // End of getValue method. + } // End of removeKeys method. RedisDBLayerIterator::RedisDBLayerIterator() { diff --git a/com.ibm.streamsx.dps/info.xml b/com.ibm.streamsx.dps/info.xml index 9298bad..e219aa8 100644 --- a/com.ibm.streamsx.dps/info.xml +++ b/com.ibm.streamsx.dps/info.xml @@ -235,7 +235,7 @@ To specifically learn how to call the DPS APIs from SPL native functions, C++ an # Reference information [../../javadoc/dps/index.html| DPS Java API Reference] - 4.1.6 + 4.1.7 4.2.0.0 diff --git a/samples/advanced/04_all_dps_apis_at_work_in_spl/com.acme.test/Main.splmm b/samples/advanced/04_all_dps_apis_at_work_in_spl/com.acme.test/Main.splmm index 2d7d6a6..472d2b9 100644 --- a/samples/advanced/04_all_dps_apis_at_work_in_spl/com.acme.test/Main.splmm +++ b/samples/advanced/04_all_dps_apis_at_work_in_spl/com.acme.test/Main.splmm @@ -13,7 +13,7 @@ use com.ibm.streamsx.lock.distributed::*; /* * This is a comprehensive test driver as well as an example application available for the DPS toolkit. It shows a way for Streams applications to have a distributed process store (DPS) for sharing data among * multiple PEs (Processing Elements) running on one or more machines. -* It supports Redis v2.8.2+ and Redis v3.x. It might also work with the other NoSQL K/V stores mentioned but does not claim support for them. +* It was tested in Redis v6.2.0 and higher versions. Running it on lower versions of Redis will make certain newer APIs to not work correctly. * You must first do a simple configuration before attempting to compile and run this test SPL application. * Please refer to this SPL project directory's etc/no-sql-kv-store-servers.cfg file. * In addition, it is recommended that you read the Redis documentation to get some tips about setting up a redis back-end. @@ -1322,25 +1322,30 @@ composite GeneralTest() printStringLn(""); printStringLn("<<<<<<< BEGIN BULK dpsGetValues API TEST >>>>>>>"); dummyRuleInstanceKey = (RuleInstanceKey_t){}; + type MyValueMap_t = map; + mutable MyValueMap_t dummyValueMap = {}; - s = dpsCreateOrGetStore("My_Dps_Get_Values_Store", dummyRuleInstanceKey, dummyRstring, err); + s = dpsCreateOrGetStore("My_Dps_Get_Values_Store", dummyRuleInstanceKey, dummyValueMap, err); if (err != 0ul) { printStringLn("Unexpected error in dpsCreateOrGetStore(My_Dps_Get_Values_Store): rc = " + (rstring)dpsGetLastStoreErrorCode() + ", msg = " + dpsGetLastStoreErrorString()); } else { // Let us add several thousands of entries in that store. mutable int32 cnt = 0; + err = 0ul; _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); while(++cnt <= 30000) { dummyRuleInstanceKey.RuleId = cnt; dummyRuleInstanceKey.NotifyByValue = "V_" + (rstring)cnt; - rstring valueStr = "MyValue_" + (rstring)cnt; - dpsPut(s, dummyRuleInstanceKey, valueStr, err); + clearM(dummyValueMap); + insertM(dummyValueMap, cnt, "Value_" + (rstring)cnt); + + dpsPut(s, dummyRuleInstanceKey, dummyValueMap, err); if(err != 0ul) { - printStringLn("Unexpected error in dpsPut(My_Dps_Get_Values__Store): rc = " + (rstring)dpsGetLastStoreErrorCode() + ", msg = " + dpsGetLastStoreErrorString() + ", cnt=" + (rstring)cnt); + printStringLn("Unexpected error in dpsPut(My_Dps_Get_Values_Store): rc = " + (rstring)dpsGetLastStoreErrorCode() + ", msg = " + dpsGetLastStoreErrorString() + ", cnt=" + (rstring)cnt); break; } } // End of while loop. @@ -1363,48 +1368,34 @@ composite GeneralTest() appendM(keys, dummyRuleInstanceKey); } + // This list will have true or false to indicate about the + // presence of each key we will send to the API to fetch its value. + // If some other operator deleted a few keys at this point in time, + // this will help us to know whether we should access or ignore the + // index in the values list for a given key. It is a convenience + // feature provided by the get values API. + mutable list keyExistsOrNot = []; + // Our value fetch results will be returned in this list. - mutable list values = []; - // Value fetch result codes will be returned in this list. - mutable list errors = []; - // The overall dpsGetValues execution result will tell us if all given keys resulted in - // a successful value fetch or one or more keys failed to fetch a value. - mutable boolean resultStatus = true; + mutable list values = []; + err = 0ul; // We can now get the values for a list of multiple keys present in the store. _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); - resultStatus = dpsGetValues(s, keys, values, errors); + dpsGetValues(s, keys, keyExistsOrNot, values, err); _timeNow = getTimestamp(); _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; int32 valuesFetched = spl.collection::size(values); - if (resultStatus != true) { - // If result status is not true, that means some of the keys - // we have didn't fetch any values. In such cases, it is necessary - // to check each element in the errors list populated by the - // DPS get values call we made above and then safely access the - // corresponding element in the values list. It is better to skip - // a values list element index that shows a non-zero error code in the - // corresponding errors list element index. + if (err != 0ul) { + // If result status is not a success, that means some or all of the keys + // we have didn't fetch any values. printStringLn(""); printStringLn(""); - // Count how many value fetch results have error codes set in them. - mutable int32 errorCodeCnt = 0; - mutable int32 idx = -1; - - for(uint64 ec in errors) { - idx++; - - if(ec != 0ul) { - ++errorCodeCnt; - printStringLn("Getting value for key " + (rstring)idx + " of " + (rstring)valuesFetched + " resulted in an error. Current error cnt = " + (rstring) errorCodeCnt + ". Current error code = " + (rstring)ec); - } - } - - printStringLn((rstring)(valuesFetched - errorCodeCnt) + " keys resulted in a successful value fetch. " + (rstring)errorCodeCnt + " keys resulted in an error to fetch a value."); + printStringLn("dpsGetValues failed. Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString()); } else { printStringLn(""); printStringLn(""); @@ -1414,7 +1405,7 @@ composite GeneralTest() printStringLn("Value for key " + (rstring)valuesFetched + " = " + (rstring)values[valuesFetched-1]); } - printStringLn("It took " + (rstring)_totalExecutionTime + " nanoseconds to get those values."); + printStringLn("It took " + (rstring)_totalExecutionTime + " nanoseconds to produce that result."); _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); @@ -1437,32 +1428,42 @@ composite GeneralTest() printStringLn("dpsRemoveStore(s) completed in " + (rstring)_totalExecutionTime + " nanoseconds."); } + clearM(keyExistsOrNot); + // As a final test for the bulk dpsGetValues test, let us try to // get the values of the store that we removed above and ensure that - // we get back full error list for all the given keys. + // we correctly get back an error. _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); - resultStatus = dpsGetValues(s, keys, values, errors); + dpsGetValues(s, keys, keyExistsOrNot, values, err); _timeNow = getTimestamp(); _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; - mutable int32 errorCnt = 0; - - for(uint64 ec in errors) { - if(ec != 0ul) { - errorCnt++; - } - } - - if(resultStatus == false) { + if(err != 0ul) { printStringLn(""); printStringLn(""); - printStringLn("We got the correct expected error result by calling dpsGetValues(s, keys) on a non-existing store. It took " + (rstring)_totalExecutionTime + " nanoseconds. Total error count = " + (rstring)errorCnt); + printStringLn("We got an error while calling dpsGetValues(s, keys) on a non-existing store. Error=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString() + ". It took " + (rstring)_totalExecutionTime + " nanoseconds."); } else { printStringLn(""); printStringLn(""); - printStringLn("We got a wrong result from dpsGetValues returning values from a non-existing store. It took " + (rstring)_totalExecutionTime + " nanoseconds."); + mutable boolean correctResult = true; + + // This should have returned every key as non-existing. + for(boolean x in keyExistsOrNot) { + if(x == true) { + // This is not correct. + // We can't have any key present in a removed store. + correctResult = false; + break; + } + } + + if(correctResult == true) { + printStringLn("keyExistsOrNot size = " + (rstring)spl.collection::size(keyExistsOrNot) + ". We got a correct expected result from running dpsGetValues on a non-existing store. It took " + (rstring)_totalExecutionTime + " nanoseconds."); + } else { + printStringLn("keyExistsOrNot size = " + (rstring)spl.collection::size(keyExistsOrNot) + ". We got a wrong result from dpsGetValues returning values from a non-existing store. It took " + (rstring)_totalExecutionTime + " nanoseconds."); + } } } @@ -1512,11 +1513,6 @@ composite GeneralTest() mutable list keys = []; // Our value fetch results will be returned in this list. mutable list values = []; - // Value fetch result codes will be returned in this list. - mutable list errors = []; - // The overall dpsGetKVPairs execution result will tell us if all given keys resulted in - // a successful value fetch or one or more keys failed to fetch a value. - mutable boolean resultStatus = true; // Stay in a loop and read multiple key/value pairs in 5 different blocks. // Block1: 15K k/V pairs, Block2: 18K K/V pairs, Block3: 21K K/V pairs, @@ -1524,23 +1520,24 @@ composite GeneralTest() list numberOfPairsNeeded = [15000, 18000, 21000, 24000, 27000]; mutable int32 keyStartPosition = 0; - for(int32 myLoopCnt in range(5)) { + for(int32 myLoopCnt in range(5)) { + err = 0ul; _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); - resultStatus = dpsGetKVPairs(s, keys, values, keyStartPosition, numberOfPairsNeeded[myLoopCnt], errors); + dpsGetKVPairs(s, keys, values, keyStartPosition, numberOfPairsNeeded[myLoopCnt], err); _timeNow = getTimestamp(); _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; - if (resultStatus != true) { + if (err != 0ul) { printStringLn(""); printStringLn(""); - printStringLn("Unexpected error in dpsGetKVPairs(s, keys): K/V pair fetch failed for one or more keys. You can check the errors list for which key(s) failed to fetch its value."); + printStringLn("Error in dpsGetKVPairs(s, keys): keyStartPosition=" + (rstring)keyStartPosition + ", pairsNeeded=" + (rstring)numberOfPairsNeeded[myLoopCnt] + ", Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString() + ", cnt=" + (rstring)myLoopCnt); } else { printStringLn(""); printStringLn(""); int32 kvPairsFetched = spl.collection::size(keys); - printStringLn("Result for Bulk Get KV Pairs " + (rstring)(myLoopCnt + 1) + ": keyStartPosition=" + (rstring)keyStartPosition + ", numberOfPairsNeeded=" + (rstring)numberOfPairsNeeded[myLoopCnt] + ". dpsGetKVPairs fetched " + (rstring)kvPairsFetched + " k/v pairs in " + (rstring)_totalExecutionTime + " nanoseconds."); + printStringLn("Result for Bulk Get KV Pairs " + (rstring)(myLoopCnt + 1) + ": keyStartPosition=" + (rstring)keyStartPosition + ", numberOfPairsNeeded=" + (rstring)numberOfPairsNeeded[myLoopCnt] + ". dpsGetKVPairs fetched " + (rstring)kvPairsFetched + " K/V pairs in " + (rstring)_totalExecutionTime + " nanoseconds."); // Just for sanity test, we will print the very first and last K/V pair in the fetched block. if(kvPairsFetched > 0) { @@ -1559,15 +1556,15 @@ composite GeneralTest() // Let us now give the number of pairs needed as 0 and see if it fetches a maximum of 50000 key/value pairs. _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); - resultStatus = dpsGetKVPairs(s, keys, values, 0, 0, errors); + dpsGetKVPairs(s, keys, values, 0, 0, err); _timeNow = getTimestamp(); _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; - if (resultStatus != true) { + if (err != 0ul) { printStringLn(""); printStringLn(""); - printStringLn("Unexpected error in dpsGetKVPairs(s, keys) for 50K keys: You can check the errors list for specific details."); + printStringLn("Unexpected error in dpsGetKVPairs(s, keys) for 50K keys: Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString()); } else { printStringLn(""); printStringLn(""); @@ -1583,8 +1580,8 @@ composite GeneralTest() } } - // Let us now delete 15K k/v pairs in the middle of our 60K pairs and - // make sure we can get back the remaining 45K k/v pairs. + // Let us now delete 15K K/V pairs in the middle of our 60K pairs and + // make sure we can get back the remaining 45K K/V pairs. _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); @@ -1612,22 +1609,22 @@ composite GeneralTest() // very first and last pairs are still intact with their ordering. _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); - resultStatus = dpsGetKVPairs(s, keys, values, 0, 0, errors); + dpsGetKVPairs(s, keys, values, 0, 0, err); _timeNow = getTimestamp(); _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; - if (resultStatus != true) { + if (err != 0ul) { printStringLn(""); printStringLn(""); - printStringLn("Unexpected error in dpsGetKVPairs(s, keys) for 45K keys: You can check the errors list for specific details."); + printStringLn("Unexpected error in dpsGetKVPairs(s, keys) for 45K keys: Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString()); } else { printStringLn(""); printStringLn(""); int32 kvPairsFetched = spl.collection::size(keys); printStringLn("Result for Bulk Get KV Pairs of 45K pairs : keyStartPosition=0, numberOfPairsNeeded=0. dpsGetKVPairs fetched " + (rstring)kvPairsFetched + " pairs in " + (rstring)_totalExecutionTime + " nanoseconds."); - // Just for sanity test, we will print the very first and last k/v pairs in the fetched block. + // Just for sanity test, we will print the very first and last K/V pairs in the fetched block. if(kvPairsFetched > 0) { printStringLn("Key " + (rstring)keyStartPosition + " = " + (rstring)keys[0]); printStringLn("Value " + (rstring)keyStartPosition + " = " + (rstring)values[0]); @@ -1658,11 +1655,11 @@ composite GeneralTest() } // As a final test for the bulk dpsGetKVPairs test, let us try to - // get the k/v pairs of the store that we removed above and ensure that + // get the K/V pairs of the store that we removed above and ensure that // we get back zero pairs. _timeNow = getTimestamp(); _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); - resultStatus = dpsGetKVPairs(s, keys, values, 0, 0, errors); + dpsGetKVPairs(s, keys, values, 0, 0, err); _timeNow = getTimestamp(); _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; @@ -1670,11 +1667,11 @@ composite GeneralTest() if (spl.collection::size(keys) == 0) { printStringLn(""); printStringLn(""); - printStringLn("We got the correct expected zero k/v pairs by calling dpsGetKVPairs(s, keys) on a non-existing store. It took " + (rstring)_totalExecutionTime + " nanoseconds."); + printStringLn("We got the correct expected zero K/V pairs by calling dpsGetKVPairs(s, keys) on a non-existing store. It took " + (rstring)_totalExecutionTime + " nanoseconds."); } else { printStringLn(""); printStringLn(""); - printStringLn("We got a wrong result from dpsGetKVPairs returning k/v pairs for a non-existing store. It took " + (rstring)_totalExecutionTime + " nanoseconds."); + printStringLn("We got a wrong result from dpsGetKVPairs returning K/V pairs for a non-existing store. It took " + (rstring)_totalExecutionTime + " nanoseconds."); } } @@ -1682,6 +1679,247 @@ composite GeneralTest() printStringLn("<<<<<<< END BULK dpsGetKVPairs API TEST >>>>>>>"); // <<<<<<< END BULK dpsGetKVPairs API TEST >>>>>>> + // <<<<<<< BEGIN BULK dpsPutKVPairs API TEST >>>>>>> + // Let us do a quick run to show how to put i.e. write/save K/V pairs to a given store. + // Note: This function will not attempt to make sure the existence of the given store. + // User should make sure that the store exists before using this function to do bulk writes. + printStringLn(""); + printStringLn(""); + printStringLn("<<<<<<< BEGIN BULK dpsPutKVPairs API TEST >>>>>>>"); + dummyRuleInstanceKey = (RuleInstanceKey_t){}; + + s = dpsCreateOrGetStore("My_Dps_Put_KV_Pairs_Store", dummyRuleInstanceKey, dummyValueMap, err); + + if (err != 0ul) { + printStringLn("Unexpected error in dpsCreateOrGetStore(My_Dps_Put_KV_Pairs_Store): rc = " + (rstring)dpsGetLastStoreErrorCode() + ", msg = " + dpsGetLastStoreErrorString()); + } else { + // Let us add several thousands of entries in that store via a bulk put API i.e. dpsPutKVPairs. + mutable int32 cnt = 0; + mutable list keys = []; + mutable list values = []; + mutable uint64 err = 0ul; + + // Populate the keys and values list. + while(++cnt <= 20000) { + dummyRuleInstanceKey.RuleId = cnt; + dummyRuleInstanceKey.NotifyByValue = "V_" + (rstring)cnt; + clearM(dummyValueMap); + insertM(dummyValueMap, cnt, "Value_" + (rstring)cnt); + appendM(keys, dummyRuleInstanceKey); + appendM(values, dummyValueMap); + } // End of while loop. + + _timeNow = getTimestamp(); + _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + + // Do the bulk put of the given K/V pairs. + dpsPutKVPairs(s, keys, values, err); + + _timeNow = getTimestamp(); + _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; + + if(err != 0ul) { + printStringLn("Error encountered in dpsPutKVPairs which took " + (rstring)_totalExecutionTime + " nanoseconds. Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString()); + } else { + printStringLn("dpsPutKVPairs put all the 20000 K/V pairs in " + (rstring)_totalExecutionTime + " nanoseconds."); + } + + + // Get all the values and ensure that we can find all 20K of them. + mutable list keyExistsOrNot = []; + clearM(values); + dpsGetValues(s, keys, keyExistsOrNot, values, err); + + if(err == 0ul) { + printStringLn("Verify after putKVPairs-->" + (rstring)spl.collection::size(values) + " pairs found."); + printStringLn("Key0 = " + (rstring)keys[0] + ", Value0 = " + (rstring)values[0] + ", Key19999 = " + (rstring)keys[19999] + ", Value19999 = " + (rstring)values[19999]); + } else { + printStringLn("Verify after putKVPairs-->dpsGetValues Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString()); + } + + // This test is fully done. We can get rid of this store. + dpsRemoveStore(s, err); + } + + printStringLn(""); + printStringLn("<<<<<<< END BULK dpsPutKVPairs API TEST >>>>>>>"); + // <<<<<<< END BULK dpsPutKVPairs API TEST >>>>>>> + + // <<<<<<< BEGIN BULK dpsHasKeys API TEST >>>>>>> + // Let us do a quick run to show how to check for existence of multiple keys in a given store. + // Note: This function will not attempt to make sure the existence of the given store. + // User should make sure that the store exists before using this function to do bulk key existence checks. + printStringLn(""); + printStringLn(""); + printStringLn("<<<<<<< BEGIN BULK dpsHasKeys API TEST >>>>>>>"); + dummyRuleInstanceKey = (RuleInstanceKey_t){}; + + s = dpsCreateOrGetStore("My_Dps_Has_Keys_Store", dummyRuleInstanceKey, dummyValueMap, err); + + if (err != 0ul) { + printStringLn("Unexpected error in dpsCreateOrGetStore(My_Dps_Has_Keys_Store): rc = " + (rstring)dpsGetLastStoreErrorCode() + ", msg = " + dpsGetLastStoreErrorString()); + } else { + // Let us add several thousands of entries in that store via a bulk put API i.e. dpsPutKVPairs. + mutable int32 cnt = 0; + mutable list keys = []; + mutable list values = []; + mutable uint64 err = 0ul; + + // Populate the keys and values list. + while(++cnt <= 20000) { + dummyRuleInstanceKey.RuleId = cnt; + dummyRuleInstanceKey.NotifyByValue = "V_" + (rstring)cnt; + clearM(dummyValueMap); + insertM(dummyValueMap, cnt, "Value_" + (rstring)cnt); + appendM(keys, dummyRuleInstanceKey); + appendM(values, dummyValueMap); + } // End of while loop. + + _timeNow = getTimestamp(); + _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + + // Do the bulk put of the given K/V pairs. + dpsPutKVPairs(s, keys, values, err); + + _timeNow = getTimestamp(); + _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; + + if(err != 0ul) { + printStringLn("Error encountered in dpsPutKVPairs which took " + (rstring)_totalExecutionTime + " nanoseconds. Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString()); + } else { + printStringLn("dpsPutKVPairs put all the 20000 K/V pairs in " + (rstring)_totalExecutionTime + " nanoseconds."); + } + + // Let us check for the existence of the keys we just put above. + // Before we do that, we will remove a few keys from the store to see if + // they will get reported as non-existing keys. + dpsRemove(s, keys[0], err); + dpsRemove(s, keys[4000], err); + dpsRemove(s, keys[8000], err); + dpsRemove(s, keys[12000], err); + dpsRemove(s, keys[16000], err); + + _timeNow = getTimestamp(); + _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + + mutable list results = []; + dpsHasKeys(s, keys, results, err); + + _timeNow = getTimestamp(); + _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; + + + if(err == 0ul) { + printStringLn("dpsHasKeys returned " + (rstring)spl.collection::size(results) + " key existence check results in " + (rstring)_totalExecutionTime + " nanoseconds."); + mutable int32 resultCnt = 0; + + // Let us verify if it found 19995 keys as existing and 5 as non-existing. + for(boolean x in results) { + if(x == true) { + resultCnt++; + } + } + + if(resultCnt == 19995) { + printStringLn("dpsHasKeys correctly returned 19995 keys as existing."); + } else { + printStringLn("dpsHasKeys didn't correctly return 19995 keys as existing."); + } + + // Let us verify if it found 5 specific keys as non-existing. + if(results[0] == false && results[4000] == false && results[8000] == false && + results[12000] == false && results[16000] == false) { + printStringLn("dpsHasKeys correctly returned 5 keys as non-existing."); + } else { + printStringLn("dpsHasKeys didn't correctly return 5 keys as non-existing."); + } + } else { + printStringLn("dpsHasKeys failed to do key existence checks. Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString() + ". It took " + (rstring)_totalExecutionTime + " nanoseconds."); + } + + // This test is fully done. We can get rid of this store. + dpsRemoveStore(s, err); + } + + printStringLn(""); + printStringLn("<<<<<<< END BULK dpsHasKeys API TEST >>>>>>>"); + // <<<<<<< END BULK dpsHasKeys API TEST >>>>>>> + + // <<<<<<< BEGIN BULK dpsRemoveKeys API TEST >>>>>>> + // Let us do a quick run to show how to remove multiple keys from a given store. + // Note: This function will not attempt to make sure the existence of the given store. + // User should make sure that the store exists before using this function to do bulk removal of keys. + printStringLn(""); + printStringLn(""); + printStringLn("<<<<<<< BEGIN BULK dpsRemoveKeys API TEST >>>>>>>"); + dummyRuleInstanceKey = (RuleInstanceKey_t){}; + + s = dpsCreateOrGetStore("My_Dps_Remove_Keys_Store", dummyRuleInstanceKey, dummyValueMap, err); + + if (err != 0ul) { + printStringLn("Unexpected error in dpsCreateOrGetStore(My_Dps_Remove_Keys_Store): rc = " + (rstring)dpsGetLastStoreErrorCode() + ", msg = " + dpsGetLastStoreErrorString()); + } else { + // Let us add several thousands of entries in that store via a bulk put API i.e. dpsPutKVPairs. + mutable int32 cnt = 0; + mutable list keys = []; + mutable list values = []; + mutable uint64 err = 0ul; + + // Populate the keys and values list. + while(++cnt <= 20000) { + dummyRuleInstanceKey.RuleId = cnt; + dummyRuleInstanceKey.NotifyByValue = "V_" + (rstring)cnt; + clearM(dummyValueMap); + insertM(dummyValueMap, cnt, "Value_" + (rstring)cnt); + appendM(keys, dummyRuleInstanceKey); + appendM(values, dummyValueMap); + } // End of while loop. + + _timeNow = getTimestamp(); + _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + + // Do the bulk put of the given K/V pairs. + dpsPutKVPairs(s, keys, values, err); + + _timeNow = getTimestamp(); + _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; + + if(err != 0ul) { + printStringLn("Error encountered in dpsPutKVPairs which took " + (rstring)_totalExecutionTime + " nanoseconds. Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString()); + } else { + printStringLn("dpsPutKVPairs put all the 20000 K/V pairs in " + (rstring)_totalExecutionTime + " nanoseconds."); + } + + // Let us do a removal of the keys we just put above. + _timeNow = getTimestamp(); + _timeInNanoSecondsBeforeExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + + mutable int32 totalKeysRemoved = 0; + dpsRemoveKeys(s, keys, totalKeysRemoved, err); + + _timeNow = getTimestamp(); + _timeInNanoSecondsAfterExecution = ((getSeconds(_timeNow) * (int64)1000000000) + (int64)getNanoseconds(_timeNow)); + _totalExecutionTime = _timeInNanoSecondsAfterExecution - _timeInNanoSecondsBeforeExecution; + + + if(err == 0ul) { + printStringLn("dpsRemoveKeys removed a total of " + (rstring)totalKeysRemoved + " keys in " + (rstring)_totalExecutionTime + " nanoseconds."); + } else { + printStringLn("dpsRemoveKeys failed to remove the keys. Error Code=" + (rstring)err + ", Error Message=" + dpsGetLastStoreErrorString() + ". It took " + (rstring)_totalExecutionTime + " nanoseconds."); + } + + // This test is fully done. We can get rid of this store. + dpsRemoveStore(s, err); + } + + printStringLn(""); + printStringLn("<<<<<<< END BULK dpsRemoveKeys API TEST >>>>>>>"); + // <<<<<<< END BULK dpsRemoveKeys API TEST >>>>>>> + diff --git a/samples/advanced/04_all_dps_apis_at_work_in_spl/info.xml b/samples/advanced/04_all_dps_apis_at_work_in_spl/info.xml index abb1a43..3d41f6c 100644 --- a/samples/advanced/04_all_dps_apis_at_work_in_spl/info.xml +++ b/samples/advanced/04_all_dps_apis_at_work_in_spl/info.xml @@ -4,13 +4,13 @@ 04_all_dps_apis_at_work_in_spl - 1.0.6 + 1.0.7 4.2.0.0 com.ibm.streamsx.dps - [4.1.6,9.0.0) + [4.1.7,9.0.0)