Additional string converters (#1088)

clients/mssql/values.go

@@ -10,9 +10,9 @@ import (
 	"github.com/artie-labs/transfer/lib/config/constants"
 	"github.com/artie-labs/transfer/lib/typing"
 	"github.com/artie-labs/transfer/lib/typing/columns"
+	"github.com/artie-labs/transfer/lib/typing/converters"
 	"github.com/artie-labs/transfer/lib/typing/decimal"
 	"github.com/artie-labs/transfer/lib/typing/ext"
-	"github.com/artie-labs/transfer/lib/typing/values"
 )
 
 func parseValue(colVal any, colKind columns.Column) (any, error) {
@@ -22,7 +22,7 @@ func parseValue(colVal any, colKind columns.Column) (any, error) {
 
 	boolVal, isOk := colVal.(bool)
 	if isOk {
-		colVal = values.BooleanToBit(boolVal)
+		colVal = converters.BooleanToBit(boolVal)
 	}
 
 	colValString := fmt.Sprint(colVal)

clients/snowflake/snowflake_test.go

@@ -103,7 +103,7 @@ func (s *SnowflakeTestSuite) TestExecuteMergeReestablishAuth() {
 
 	for i := 0; i < 5; i++ {
 		rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{
-			"id":         fmt.Sprintf("pk-%d", i),
+			"id":         i,
 			"created_at": time.Now().Format(time.RFC3339Nano),
 			"name":       fmt.Sprintf("Robin-%d", i),
 		}
@@ -146,7 +146,7 @@ func (s *SnowflakeTestSuite) TestExecuteMerge() {
 
 	for i := 0; i < 5; i++ {
 		rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{
-			"id":         fmt.Sprintf("pk-%d", i),
+			"id":         i,
 			"created_at": time.Now().Format(time.RFC3339Nano),
 			"name":       fmt.Sprintf("Robin-%d", i),
 		}
@@ -211,7 +211,7 @@ func (s *SnowflakeTestSuite) TestExecuteMergeDeletionFlagRemoval() {
 	rowsData := make(map[string]map[string]any)
 	for i := 0; i < 5; i++ {
 		rowsData[fmt.Sprintf("pk-%d", i)] = map[string]any{
-			"id":         fmt.Sprintf("pk-%d", i),
+			"id":         i,
 			"created_at": time.Now().Format(time.RFC3339Nano),
 			"name":       fmt.Sprintf("Robin-%d", i),
 		}

lib/typing/converters/string_converter.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/artie-labs/transfer/lib/config/constants"
 	"github.com/artie-labs/transfer/lib/typing"
+	"github.com/artie-labs/transfer/lib/typing/decimal"
 	"github.com/artie-labs/transfer/lib/typing/ext"
 )
 
@@ -16,6 +17,7 @@ type StringConverter interface {
 
 func GetStringConverter(kd typing.KindDetails) (StringConverter, error) {
 	switch kd.Kind {
+	// Time
 	case typing.Date.Kind:
 		return DateConverter{}, nil
 	case typing.Time.Kind:
@@ -26,6 +28,13 @@ func GetStringConverter(kd typing.KindDetails) (StringConverter, error) {
 		return TimestampTZConverter{}, nil
 	case typing.Array.Kind:
 		return ArrayConverter{}, nil
+	// Numbers
+	case typing.EDecimal.Kind:
+		return DecimalConverter{}, nil
+	case typing.Integer.Kind:
+		return IntegerConverter{}, nil
+	case typing.Float.Kind:
+		return FloatConverter{}, nil
 	}
 
 	// TODO: Return an error when all the types are implemented.
@@ -94,3 +103,54 @@ func (ArrayConverter) Convert(value any) (string, error) {
 
 	return string(colValBytes), nil
 }
+
+type IntegerConverter struct{}
+
+func (IntegerConverter) Convert(value any) (string, error) {
+	switch parsedVal := value.(type) {
+	case float32:
+		return Float32ToString(parsedVal), nil
+	case float64:
+		return Float64ToString(parsedVal), nil
+	case bool:
+		return fmt.Sprint(BooleanToBit(parsedVal)), nil
+	case int, int8, int16, int32, int64:
+		return fmt.Sprint(parsedVal), nil
+	default:
+		return "", fmt.Errorf("unexpected value: '%v', type: %T", value, value)
+	}
+}
+
+type FloatConverter struct{}
+
+func (FloatConverter) Convert(value any) (string, error) {
+	switch parsedVal := value.(type) {
+	case float32:
+		return Float32ToString(parsedVal), nil
+	case float64:
+		return Float64ToString(parsedVal), nil
+	case int, int8, int16, int32, int64:
+		return fmt.Sprint(parsedVal), nil
+	default:
+		return "", fmt.Errorf("unexpected value: '%v', type: %T", value, value)
+	}
+}
+
+type DecimalConverter struct{}
+
+func (DecimalConverter) Convert(value any) (string, error) {
+	switch castedColVal := value.(type) {
+	case float32:
+		return Float32ToString(castedColVal), nil
+	case float64:
+		return Float64ToString(castedColVal), nil
+	case int, int8, int16, int32, int64:
+		return fmt.Sprint(castedColVal), nil
+	case string:
+		return castedColVal, nil
+	case *decimal.Decimal:
+		return castedColVal.String(), nil
+	default:
+		return "", fmt.Errorf("unexpected value: '%v' type: %T", value, value)
+	}
+}

lib/typing/converters/string_converter_test.go

@@ -3,11 +3,14 @@ package converters
 import (
 	"testing"
 
-	"github.com/artie-labs/transfer/lib/config/constants"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/artie-labs/transfer/lib/config/constants"
+	"github.com/artie-labs/transfer/lib/numbers"
+	"github.com/artie-labs/transfer/lib/typing/decimal"
 )
 
-func TestArrayConverter(t *testing.T) {
+func TestArrayConverter_Convert(t *testing.T) {
 	// Array
 	{
 		// Normal arrays
@@ -22,3 +25,82 @@ func TestArrayConverter(t *testing.T) {
 		assert.Equal(t, `["__debezium_unavailable_value"]`, val)
 	}
 }
+
+func TestFloatConverter_Convert(t *testing.T) {
+	{
+		// Unexpected type
+		_, err := FloatConverter{}.Convert("foo")
+		assert.ErrorContains(t, err, `unexpected value: 'foo', type: string`)
+	}
+	{
+		// Float32
+		val, err := FloatConverter{}.Convert(float32(123.45))
+		assert.NoError(t, err)
+		assert.Equal(t, "123.45", val)
+	}
+	{
+		// Float64
+		val, err := FloatConverter{}.Convert(float64(123.45))
+		assert.NoError(t, err)
+		assert.Equal(t, "123.45", val)
+	}
+	{
+		// Integers
+		for _, input := range []any{42, int8(42), int16(42), int32(42), int64(42), float32(42), float64(42)} {
+			val, err := FloatConverter{}.Convert(input)
+			assert.NoError(t, err)
+			assert.Equal(t, "42", val)
+		}
+	}
+}
+
+func TestIntegerConverter_Convert(t *testing.T) {
+	{
+		// Various numbers
+		for _, val := range []any{42, int8(42), int16(42), int32(42), int64(42), float32(42), float64(42)} {
+			parsedVal, err := IntegerConverter{}.Convert(val)
+			assert.NoError(t, err)
+			assert.Equal(t, "42", parsedVal)
+		}
+	}
+	{
+		// Booleans
+		{
+			// True
+			val, err := IntegerConverter{}.Convert(true)
+			assert.NoError(t, err)
+			assert.Equal(t, "1", val)
+		}
+		{
+			// False
+			val, err := IntegerConverter{}.Convert(false)
+			assert.NoError(t, err)
+			assert.Equal(t, "0", val)
+		}
+	}
+}
+
+func TestDecimalConverter_Convert(t *testing.T) {
+	{
+		// Extended decimal
+		val, err := DecimalConverter{}.Convert(decimal.NewDecimal(numbers.MustParseDecimal("123.45")))
+		assert.NoError(t, err)
+		assert.Equal(t, "123.45", val)
+	}
+	{
+		// Floats
+		for _, input := range []any{float32(123.45), float64(123.45)} {
+			val, err := DecimalConverter{}.Convert(input)
+			assert.NoError(t, err)
+			assert.Equal(t, "123.45", val)
+		}
+	}
+	{
+		// Integers
+		for _, input := range []any{42, int8(42), int16(42), int32(42), int64(42), float32(42), float64(42)} {
+			val, err := DecimalConverter{}.Convert(input)
+			assert.NoError(t, err)
+			assert.Equal(t, "42", val)
+		}
+	}
+}

lib/typing/converters/util.go

@@ -0,0 +1,19 @@
+package converters
+
+import "strconv"
+
+func Float64ToString(value float64) string {
+	return strconv.FormatFloat(value, 'f', -1, 64)
+}
+
+func Float32ToString(value float32) string {
+	return strconv.FormatFloat(float64(value), 'f', -1, 32)
+}
+
+func BooleanToBit(val bool) int {
+	if val {
+		return 1
+	} else {
+		return 0
+	}
+}

lib/typing/converters/util_test.go

@@ -0,0 +1,68 @@
+package converters
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBooleanToBit(t *testing.T) {
+	assert.Equal(t, 1, BooleanToBit(true))
+	assert.Equal(t, 0, BooleanToBit(false))
+}
+
+func TestFloat32ToString(t *testing.T) {
+	type ioPair struct {
+		input  float32
+		output string
+	}
+
+	ioPairs := []ioPair{
+		{123.456, "123.456"},
+		{0.0, "0"},
+		{-1.0, "-1"},
+		{1.0, "1"},
+		{340282350000000000000000000000000000000, "340282350000000000000000000000000000000"},
+		{math.MaxFloat32, "340282350000000000000000000000000000000"},
+		{0.000000000000000000000000000000000000000000001, "0.000000000000000000000000000000000000000000001"},
+		{-340282350000000000000000000000000000000, "-340282350000000000000000000000000000000"},
+		{1.401298464324817070923729583289916131280e-45, "0.000000000000000000000000000000000000000000001"},
+		{1.17549435e-38, "0.000000000000000000000000000000000000011754944"},
+		{-1.17549435e-38, "-0.000000000000000000000000000000000000011754944"},
+		{2.71828, "2.71828"},
+		{-2.71828, "-2.71828"},
+		{3.14159, "3.14159"},
+		{-3.14159, "-3.14159"},
+	}
+
+	for _, pair := range ioPairs {
+		assert.Equal(t, pair.output, Float32ToString(pair.input), pair.input)
+	}
+}
+
+func TestFloat64ToString(t *testing.T) {
+	type ioPair struct {
+		input  float64
+		output string
+	}
+
+	ioPairs := []ioPair{
+		{123.456, "123.456"},
+		{0.0, "0"},
+		{-1.0, "-1"},
+		{1.0, "1"},
+		{1.7976931348623157e+308, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
+		{math.MaxFloat64, "179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
+		{4.9406564584124654e-324, "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"},
+		{-1.7976931348623157e+308, "-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
+		{2.718281828459045, "2.718281828459045"},
+		{-2.718281828459045, "-2.718281828459045"},
+		{3.141592653589793, "3.141592653589793"},
+		{-3.141592653589793, "-3.141592653589793"},
+	}
+
+	for _, pair := range ioPairs {
+		assert.Equal(t, pair.output, Float64ToString(pair.input), pair.input)
+	}
+}

lib/typing/values/string.go

@@ -4,32 +4,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
-	"strconv"
 	"strings"
 
 	"github.com/artie-labs/transfer/lib/config/constants"
 	"github.com/artie-labs/transfer/lib/stringutil"
 	"github.com/artie-labs/transfer/lib/typing"
 	"github.com/artie-labs/transfer/lib/typing/converters"
-	"github.com/artie-labs/transfer/lib/typing/decimal"
 )
 
-func Float64ToString(value float64) string {
-	return strconv.FormatFloat(value, 'f', -1, 64)
-}
-
-func Float32ToString(value float32) string {
-	return strconv.FormatFloat(float64(value), 'f', -1, 32)
-}
-
-func BooleanToBit(val bool) int {
-	if val {
-		return 1
-	} else {
-		return 0
-	}
-}
-
 func ToString(colVal any, colKind typing.KindDetails) (string, error) {
 	if colVal == nil {
 		return "", fmt.Errorf("colVal is nil")
@@ -45,7 +27,6 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) {
 	}
 
 	// TODO: Move all of this into converter function
-
 	switch colKind.Kind {
 	case typing.String.Kind:
 		isArray := reflect.ValueOf(colVal).Kind() == reflect.Slice
@@ -78,35 +59,6 @@ func ToString(colVal any, colKind typing.KindDetails) (string, error) {
 				return string(colValBytes), nil
 			}
 		}
-	case typing.Float.Kind:
-		switch parsedVal := colVal.(type) {
-		case float32:
-			return Float32ToString(parsedVal), nil
-		case float64:
-			return Float64ToString(parsedVal), nil
-		}
-	case typing.Integer.Kind:
-		switch parsedVal := colVal.(type) {
-		case float32:
-			return Float32ToString(parsedVal), nil
-		case float64:
-			return Float64ToString(parsedVal), nil
-		case bool:
-			return fmt.Sprint(BooleanToBit(parsedVal)), nil
-		}
-	case typing.EDecimal.Kind:
-		switch castedColVal := colVal.(type) {
-		// It's okay if it's not a *decimal.Decimal, so long as it's a float or string.
-		// By having the flexibility of handling both *decimal.Decimal and float64/float32/string values within the same batch will increase our ability for data digestion.
-		case int64, int32, float64, float32:
-			return fmt.Sprint(castedColVal), nil
-		case string:
-			return castedColVal, nil
-		case *decimal.Decimal:
-			return castedColVal.String(), nil
-		}
-
-		return "", fmt.Errorf("unexpected colVal type: %T", colVal)
 	}
 
 	return fmt.Sprint(colVal), nil