Merge pull request #15 from Kami/exclude_from_index_support

Add support by excluding specific fields from index

.travis.yml

@@ -12,6 +12,23 @@ branches:
   only:
   - master
 
+matrix:
+  fast_finish: false
+  include:
+    - name: Unit Tests
+      script:
+        - cd datastore-translator
+        - go test -v ./... -race -coverprofile=coverage.txt -covermode=atomic
+    - name: Integration Tests
+      env:
+        - DATASTORE_EMULATOR_HOST=127.0.0.1:8081
+        - DATASTORE_EMULATOR_HOST_PATH=127.0.0.1:8081/datastore
+        - DATASTORE_HOST=http://127.0.0.1:8081
+        - DATASTORE_PROJECT_ID=translator-tests
+      script:
+        - cd datastore-translator
+        - go test -v ./... -race -tags=integration
+
 notifications:
   email:
     on_success: change
@@ -20,19 +37,7 @@ notifications:
 install: true
 
 before_script:
-  - export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)"
-  - echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-  - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-  - sudo apt-get update && sudo apt-get install google-cloud-sdk
-  - nohup gcloud beta emulators datastore start &
-  - sleep 30
-  - export DATASTORE_EMULATOR_HOST=localhost:8081
-
-script:
-  - cd datastore-translator
-  - echo $DATASTORE_EMULATOR_HOST
-  - go test -v ./... -race -coverprofile=coverage.txt -covermode=atomic
-#  - go test -v ./... -race -tags=integration
+  - ./scripts/run-datastore-emulator.sh
 
 after_success:
   - bash <(curl -s https://codecov.io/bash)

README.md

@@ -9,7 +9,7 @@
 
 This is largely inspired from being able to persist protocol buffers to Google Cloud Datastore. Protobuf messages that the datstore supports are listed [here](https://github.com/googleapis/googleapis/blob/c50d9e822e19e069b7e3758736ea58cb4f35267c/google/datastore/v1/entity.proto#L188).
 
-This repository acts as a translator to translate any given ``proto.Message`` to ``datastore.Entity`` that the datastore understands and 
+This repository acts as atranslator to translate any given ``proto.Message`` to ``datastore.Entity`` that the datastore understands and 
 ``datastore.Entity`` to any ``proto.Message``.
 
 This repository also addresses some of the limitations that [google-cloud-go](https://github.com/googleapis/google-cloud-go/tree/master/datastore) has.
@@ -62,7 +62,7 @@ func main() {
 	}
 
 	// 3. translate the protobuf message to the format that datastore understands
-	entity, err := translator.ProtoMessageToDatastoreEntity(execReq, true)
+	entity, err := translator.ProtoMessageToDatastoreEntity(execReq, true, nil)
 	if err != nil {
 		log.Fatalf("unable to translate execution request to datastore format, error: %v", err)
 	}
@@ -88,7 +88,7 @@ func main() {
 	dsExecReq := &execution.ExecutionRequest{}
 
 	// 8. convert the value fetched from datastore to protobuf
-	err = translator.DatastoreEntityToProtoMessage(dsEntity,dsExecReq, true)
+	err = translator.DatastoreEntityToProtoMessage(dsEntity,dsExecReq, true, nil)
 	if err != nil {
 		log.Fatalf("error while converting to proto message, %v", err)
 	}

datastore-translator/translator.go

@@ -18,15 +18,15 @@ import (
 )
 
 // ProtoMessageToDatastoreEntity will generate an Entity Protobuf that datastore understands
-func ProtoMessageToDatastoreEntity(src proto.Message, snakeCase bool) (entity datastore.Entity, err error) {
+func ProtoMessageToDatastoreEntity(src proto.Message, snakeCase bool, excludeFromIndex []string) (entity datastore.Entity, err error) {
 	srcValues := reflect.ValueOf(src).Elem()
 	properties := make(map[string]*datastore.Value)
 
 	for i := 0; i < srcValues.NumField(); i++ {
 		fName := srcValues.Type().Field(i).Name
 		if !strings.ContainsAny(fName, "XXX_") {
 			var value *datastore.Value
-			if value, err = toDatastoreValue(srcValues.Field(i), snakeCase); err != nil {
+			if value, err = toDatastoreValue(fName, srcValues.Field(i), snakeCase, excludeFromIndex); err != nil {
 				return
 			} else {
 				if value != nil {
@@ -117,7 +117,18 @@ func DatastoreEntityToProtoMessage(src *datastore.Entity, dst proto.Message, sna
 	return err
 }
 
-func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, error) {
+func toDatastoreValue(fName string, fValue reflect.Value, snakeCase bool, excludeFromIndex []string) (*datastore.Value, error) {
+	// NOTE: excludeFieldFromIndex needs to contain original Protobuf model field names which can also include underscores
+	var origFName string
+
+	if snakeCase {
+		origFName = toSnakeCase(fName)
+	} else {
+		origFName = fName
+	}
+
+	excludeFieldFromIndex := contains(excludeFromIndex, origFName)
+
 	value := &datastore.Value{}
 	var err error
 	switch fValue.Kind() {
@@ -148,7 +159,7 @@ func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, e
 			size := fValue.Len()
 			values := make([]*datastore.Value, 0)
 			for i := 0; i < size; i++ {
-				val, err := toDatastoreValue(fValue.Index(i), snakeCase)
+				val, err := toDatastoreValue(fName, fValue.Index(i), snakeCase, nil)
 				if err != nil {
 					return nil, err
 				}
@@ -167,7 +178,7 @@ func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, e
 		for _, key := range mapValues.MapKeys() {
 			k := fmt.Sprint(key)
 			//TODO what if there is an error?
-			v, _ := toDatastoreValue(mapValues.MapIndex(key), snakeCase)
+			v, _ := toDatastoreValue(fName, mapValues.MapIndex(key), snakeCase, nil)
 			//fmt.Printf("key; %v, value: %v\n",k,v)
 			properties[k] = v
 		}
@@ -203,7 +214,7 @@ func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, e
 			// translate any imported protobuf's that we defined ourself
 			if !fValue.IsNil() && fValue.IsValid() {
 				if importedProto, ok := reflect.ValueOf(fValue.Interface()).Interface().(proto.Message); ok {
-					entityOfImportedProto, err := ProtoMessageToDatastoreEntity(importedProto, snakeCase)
+					entityOfImportedProto, err := ProtoMessageToDatastoreEntity(importedProto, snakeCase, excludeFromIndex)
 					if err != nil {
 						return nil, err
 					}
@@ -220,6 +231,13 @@ func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, e
 		log.Println(errString)
 		err = errors.New(errString)
 	}
+
+	if excludeFieldFromIndex == true {
+		value.ExcludeFromIndexes = true
+	} else {
+		value.ExcludeFromIndexes = false
+	}
+
 	return value, err
 }
 
@@ -311,3 +329,12 @@ func toSnakeCase(name string) string {
 	snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
 	return strings.ToLower(snake)
 }
+
+func contains(a []string, x string) bool {
+	for _, n := range a {
+		if x == n {
+			return true
+		}
+	}
+	return false
+}

datastore-translator/translator_integration_test.go

@@ -11,14 +11,21 @@ import (
 	"gotest.tools/assert"
 	"log"
 	"testing"
+	"time"
 )
 
+const DATASTORE_CONNECT_TIMEOUT = 5 * time.Second
+
 func TestIntegration(t *testing.T) {
 	ctx := context.Background()
+
 	// 1. create a new datastore client
 	client, err := datastore.NewClient(ctx, "st2-saas-prototype-dev")
 	assert.NilError(t, err)
 
+	ctx, cancel := context.WithTimeout(ctx, DATASTORE_CONNECT_TIMEOUT)
+	defer cancel()
+
 	// 2. create a key that we plan to save into
 	key := datastore.NameKey("Example_DB_Model", "complex_proto_2", nil)
 
@@ -60,7 +67,7 @@ func TestIntegration(t *testing.T) {
 
 	log.Printf("original proto: %v", srcProto)
 	// 4. translate the source protobuf to datastore.Entity
-	translatedSrcProto, err := ProtoMessageToDatastoreEntity(srcProto, true)
+	translatedSrcProto, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
 	assert.NilError(t, err)
 
 	// 5. save the translated protobuf to datastore
@@ -120,11 +127,14 @@ func TestEmptyProtoMessage(t *testing.T) {
 	client, err := datastore.NewClient(ctx, "st2-saas-prototype-dev")
 	assert.NilError(t, err)
 
+	ctx, cancel := context.WithTimeout(ctx, DATASTORE_CONNECT_TIMEOUT)
+	defer cancel()
+
 	// 2. create a key that we plan to save into
 	key := datastore.NameKey("Example_DB_Model", "complex_proto_empty", nil)
 
 	srcProto := &example.ExampleDBModel{}
-	translatedProto, err := ProtoMessageToDatastoreEntity(srcProto, false)
+	translatedProto, err := ProtoMessageToDatastoreEntity(srcProto, false, nil)
 	assert.NilError(t, err)
 
 	_, err = client.PutEntity(ctx, key, &translatedProto)
@@ -149,13 +159,16 @@ func TestProtoWithNilPointer(t *testing.T) {
 	client, err := datastore.NewClient(ctx, "st2-saas-prototype-dev")
 	assert.NilError(t, err)
 
+	ctx, cancel := context.WithTimeout(ctx, DATASTORE_CONNECT_TIMEOUT)
+	defer cancel()
+
 	// 2. create a key that we plan to save into
 	key := datastore.NameKey("Example_DB_Model", "complex_proto_empty", nil)
 
 	srcProto := &example.ExampleDBModel{
 		TimestampKey: ptypes.TimestampNow(),
 	}
-	translatedProto, err := ProtoMessageToDatastoreEntity(srcProto, false)
+	translatedProto, err := ProtoMessageToDatastoreEntity(srcProto, false, nil)
 	assert.NilError(t, err)
 
 	_, err = client.PutEntity(ctx, key, &translatedProto)

datastore-translator/translator_test.go

@@ -16,7 +16,7 @@ func TestNestedModel(t *testing.T) {
 		StringKey: "some random string",
 		Int32Key:  22,
 	}
-	entity, err := ProtoMessageToDatastoreEntity(srcProto, true)
+	entity, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
 	// make sure there is no error
 	assert.NilError(t, err)
 	dstProto := &example.ExampleNestedModel{}
@@ -27,6 +27,42 @@ func TestNestedModel(t *testing.T) {
 	assert.DeepEqual(t, srcProto.GetStringKey(), dstProto.GetStringKey())
 }
 
+func TestProtoMessageToDatastoreEntityWithExcludeFieldsFromIndex(t *testing.T) {
+	srcProto := &example.ExampleDBModel{
+		StringKey: "some random string key for testing",
+		BoolKey:   true,
+		Int32Key:  int32(32),
+		Int64Key:  64,
+		FloatKey:  float32(10.14),
+		DoubleKey: float64(10.2121),
+		BytesKey:  []byte("this is a byte array"),
+	}
+
+	// No exclude from index fields specified
+	entity, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
+	assert.NilError(t, err)
+	assert.Equal(t, entity.Properties["string_key"].ExcludeFromIndexes, false)
+	assert.Equal(t, entity.Properties["bytes_key"].ExcludeFromIndexes, false)
+	assert.Equal(t, entity.Properties["float_key"].ExcludeFromIndexes, false)
+
+	assert.Equal(t, entity.Properties["bool_key"].ExcludeFromIndexes, false)
+	assert.Equal(t, entity.Properties["int32_key"].ExcludeFromIndexes, false)
+	assert.Equal(t, entity.Properties["double_key"].ExcludeFromIndexes, false)
+
+	// Some exclude from index fields specified
+	excludeFromIndex := [...]string{"string_key", "bytes_key", "float_key"}
+
+	entity, err = ProtoMessageToDatastoreEntity(srcProto, true, excludeFromIndex[:])
+	assert.NilError(t, err)
+	assert.Equal(t, entity.Properties["string_key"].ExcludeFromIndexes, true)
+	assert.Equal(t, entity.Properties["bytes_key"].ExcludeFromIndexes, true)
+	assert.Equal(t, entity.Properties["float_key"].ExcludeFromIndexes, true)
+
+	assert.Equal(t, entity.Properties["bool_key"].ExcludeFromIndexes, false)
+	assert.Equal(t, entity.Properties["int32_key"].ExcludeFromIndexes, false)
+	assert.Equal(t, entity.Properties["double_key"].ExcludeFromIndexes, false)
+}
+
 func TestFullyPopulatedModel(t *testing.T) {
 	srcProto := &example.ExampleDBModel{
 		StringKey: "some random string key for testing",
@@ -73,7 +109,7 @@ func TestFullyPopulatedModel(t *testing.T) {
 		},
 		TimestampKey: ptypes.TimestampNow(),
 	}
-	entity, err := ProtoMessageToDatastoreEntity(srcProto, true)
+	entity, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
 
 	// make sure there is no error
 	assert.NilError(t, err)
@@ -125,7 +161,7 @@ func TestPartialModel(t *testing.T) {
 			"struct-key-null":   {Kind: &structpb.Value_NullValue{}},
 		},
 	}
-	entity, err := ProtoMessageToDatastoreEntity(partialProto, true)
+	entity, err := ProtoMessageToDatastoreEntity(partialProto, true, nil)
 	assert.NilError(t, err, err)
 	log.Println(entity)
 	dstProto := &structpb.Struct{}
@@ -140,7 +176,7 @@ func TestUnSupportedTypes(t *testing.T) {
 	srcProto := &unsupported.Model{
 		Uint32Key: uint32(10),
 	}
-	_, err := ProtoMessageToDatastoreEntity(srcProto, false)
+	_, err := ProtoMessageToDatastoreEntity(srcProto, false, nil)
 	assert.Error(t, err, "[toDatastoreValue]: datatype[uint32] not supported")
 }
 
@@ -149,7 +185,7 @@ func TestPMtoDE(t *testing.T) {
 		StringKey: "some random string",
 		Int32Key:  22,
 	}
-	entity, err := ProtoMessageToDatastoreEntity(srcProto, true)
+	entity, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
 	assert.NilError(t, err)
 	log.Println(entity)
 }
@@ -317,7 +353,7 @@ func TestProtoWithCustomImport(t *testing.T) {
 		},
 	}
 
-	dstEntity, err := ProtoMessageToDatastoreEntity(srcProto, false)
+	dstEntity, err := ProtoMessageToDatastoreEntity(srcProto, false, nil)
 	assert.NilError(t, err)
 	// our interest here only to compare the ComplexArrayKey
 	assert.DeepEqual(t, srcEntity.GetProperties()["ComplexArrayKey"], dstEntity.GetProperties()["ComplexArrayKey"])

datastore-translator/ts_struct_test.go

@@ -15,7 +15,7 @@ func TestAddTSSupport(t *testing.T) {
 		StartedOn: ptypes.TimestampNow(),
 	}
 	log.Println("Source: ", src)
-	srcEntity, err := ProtoMessageToDatastoreEntity(src, false)
+	srcEntity, err := ProtoMessageToDatastoreEntity(src, false, nil)
 
 	assert.NilError(t, err)
 	log.Println("Source Datastore Entity: ", srcEntity)
@@ -50,7 +50,7 @@ func TestAddStructSupport(t *testing.T) {
 		},
 	}
 	log.Println("Source: ", src)
-	srcEntity, err := ProtoMessageToDatastoreEntity(src, false)
+	srcEntity, err := ProtoMessageToDatastoreEntity(src, false, nil)
 
 	assert.NilError(t, err)
 	log.Println("Source Datastore Entity: ", srcEntity)
@@ -75,7 +75,7 @@ func TestSliceofNestedMessages(t *testing.T) {
 		},
 	}
 	log.Println("Source: ", src)
-	srcEntity, err := ProtoMessageToDatastoreEntity(src, false)
+	srcEntity, err := ProtoMessageToDatastoreEntity(src, false, nil)
 	assert.NilError(t, err)
 
 	log.Println("Source Datastore Entity: ", srcEntity)
@@ -97,7 +97,7 @@ func TestNestedMessages(t *testing.T) {
 		},
 	}
 	log.Println("Source: ", src)
-	srcEntity, err := ProtoMessageToDatastoreEntity(src, false)
+	srcEntity, err := ProtoMessageToDatastoreEntity(src, false, nil)
 	assert.NilError(t, err)
 
 	log.Println("Source Datastore Entity: ", srcEntity)

scripts/run-datastore-emulator.sh

@@ -11,7 +11,7 @@ gcloud beta emulators datastore start --host-port=127.0.0.1:8081 --no-store-on-d
 EMULATOR_PID=$!
 
 # Give process some time to start up
-sleep 25
+sleep 5
 
 if ps -p ${EMULATOR_PID} > /dev/null; then
     echo "google cloud datastore emulator successfully started"
@@ -21,4 +21,4 @@ else
     echo "Failed to start google cloud datastore emulator"
     tail -30 /tmp/emulator.log
     exit 1
-fi
+fi