diff --git a/bin/debug.sh b/bin/debug.sh index 0cf5dd3..6a19560 100755 --- a/bin/debug.sh +++ b/bin/debug.sh @@ -19,23 +19,24 @@ : ${ERROR_PATH:='/tmp/spooldir/error'} : ${FINISHED_PATH:='/tmp/spooldir/finished'} : ${DEBUG_SUSPEND_FLAG:='y'} -export KAFKA_DEBUG='n' -export KAFKA_OPTS='-agentpath:/Applications/YourKit-Java-Profiler-2017.02.app/Contents/Resources/bin/mac/libyjpagent.jnilib=disablestacktelemetry,exceptions=disable,delay=10000' +export KAFKA_DEBUG='y' +export DEBUG_SUSPEND_FLAG='y' +# export KAFKA_OPTS='-agentpath:/Applications/YourKit-Java-Profiler-2017.02.app/Contents/Resources/bin/mac/libyjpagent.jnilib=disablestacktelemetry,exceptions=disable,delay=10000' set -e -mvn clean package +# mvn clean package -#if [ ! -d "${INPUT_PATH}" ]; then -# mkdir -p "${INPUT_PATH}" -#fi +if [ ! -d "${INPUT_PATH}" ]; then + mkdir -p "${INPUT_PATH}" +fi -#if [ ! -d "${ERROR_PATH}" ]; then -# mkdir -p "${ERROR_PATH}" -#fi +if [ ! -d "${ERROR_PATH}" ]; then + mkdir -p "${ERROR_PATH}" +fi -#if [ ! -d "${FINISHED_PATH}" ]; then -# mkdir -p "${FINISHED_PATH}" -#fi +if [ ! -d "${FINISHED_PATH}" ]; then + mkdir -p "${FINISHED_PATH}" +fi -#cp src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.data "${INPUT_PATH}/FieldsMatch.csv -connect-standalone config/connect-avro-docker.properties config/ELFTesting.properties \ No newline at end of file +cp src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.data "${INPUT_PATH}/FieldsMatch.csv" +connect-standalone config/connect-avro-docker.properties config/CSVSchemaGenerator.properties \ No newline at end of file diff --git a/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/CsvSchemaGenerator.java b/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/CsvSchemaGenerator.java index fcca3a0..5440ccd 100644 --- a/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/CsvSchemaGenerator.java +++ b/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/CsvSchemaGenerator.java @@ -16,6 +16,7 @@ package com.github.jcustenborder.kafka.connect.spooldir; import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; import com.opencsv.ICSVParser; import org.apache.kafka.connect.data.Schema; import org.slf4j.Logger; @@ -44,7 +45,8 @@ protected Map determineFieldTypes(InputStream inputStream) Map typeMap = new LinkedHashMap<>(); ICSVParser parserBuilder = this.config.createCSVParserBuilder(); try (InputStreamReader reader = new InputStreamReader(inputStream)) { - try (CSVReader csvReader = new CSVReader(reader, 0, parserBuilder)) { + CSVReaderBuilder readerBuilder = this.config.createCSVReaderBuilder(reader, parserBuilder); + try (CSVReader csvReader = readerBuilder.build()) { String[] headers = null; if (this.config.firstRowAsHeader) { diff --git a/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceTask.java b/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceTask.java index 4abab56..90e5071 100644 --- a/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceTask.java +++ b/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceTask.java @@ -91,7 +91,8 @@ public long recordOffset() { if (null == this.csvReader) { result = -1L; } else { - result = this.csvReader.getLinesRead(); + result = this.csvReader.getLinesRead() - this.config.skipLines - + (this.config.firstRowAsHeader ? 1 : 0); } return result; } @@ -103,7 +104,10 @@ public List process() throws IOException { while (records.size() < this.config.batchSize) { String[] row = this.csvReader.readNext(); - if (row == null) { + if (null == row) { + break; + } + if (row.length == 1 && null == row[0]) { break; } log.trace("process() - Row on line {} has {} field(s)", recordOffset(), row.length); diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/BlankLines.data b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/BlankLines.data new file mode 100644 index 0000000..6db7818 --- /dev/null +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/BlankLines.data @@ -0,0 +1,25 @@ +id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color +1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313 +2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf +3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN, +4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414 +5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN, +6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR, +7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d +8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU, +9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN, +10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9 +11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36 +12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909 +13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN, +14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943 +15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119 +16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH, +17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID, +18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN, +19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL, +20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128 + + + + \ No newline at end of file diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/BlankLines.json b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/BlankLines.json new file mode 100644 index 0000000..11ad315 --- /dev/null +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/BlankLines.json @@ -0,0 +1,5381 @@ +{ + "settings" : { + "csv.first.row.as.header" : "true" + }, + "offset" : { }, + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "expected" : [ { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 1 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Jack" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Garcia" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jgarcia0@shop-pro.jp" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "196.56.44.185" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1443626943000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 347.77 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "IT" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#4a2313" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 2 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 2 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 2 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "John" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Kim" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jkim1@miibeian.gov.cn" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "53.19.132.185" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1447497249000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 251.24 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CZ" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#3e56cf" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 2 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 3 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 3 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 3 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Ashley" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Austin" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "aaustin2@hatena.ne.jp" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "21.164.37.9" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + } + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 819.47 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 3 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 4 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 4 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 4 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Jonathan" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Mcdonald" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jmcdonald3@amazon.co.uk" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "188.172.42.140" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1451313421000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 868.38 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ID" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#1b1414" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 4 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 5 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 5 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 5 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Helen" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Lane" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "hlane4@trellian.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "159.171.138.190" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1467312078000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 398.97 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "TN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 5 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 6 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 6 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 6 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Scott" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Lopez" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "slopez5@google.co.jp" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "86.194.226.35" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1439432031000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 322.99 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "BR" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 6 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 7 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 7 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 7 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Christine" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Franklin" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "cfranklin6@reuters.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "248.173.207.64" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1450783797000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 301.26 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "PH" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#1d5e9d" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 7 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 8 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 8 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 8 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Helen" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Andrews" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "handrews7@histats.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "83.160.63.181" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1457264470000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 217.96 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CU" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 8 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 9 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 9 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 9 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Stephanie" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Gordon" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "sgordon8@goodreads.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "193.143.42.212" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1445983644000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 495.80 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 9 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 10 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 10 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 10 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Shirley" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Andrews" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "sandrews9@flickr.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "99.113.183.206" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1446894772000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 157.75 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "BR" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#fc1da9" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 10 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 11 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 11 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 11 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Joshua" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Reid" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jreida@wikia.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "197.96.118.164" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1440249378000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 431.80 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CO" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#6e3e36" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 11 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 12 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 12 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 12 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Frances" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Parker" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "fparkerb@engadget.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "226.237.57.25" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1445133015000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 188.21 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "BR" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#73e909" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 12 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 13 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 13 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 13 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Sharon" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Lawson" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "slawsonc@bravesites.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "198.189.134.106" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1452793869000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 206.73 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "VN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 13 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 14 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 14 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 14 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Elizabeth" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Wells" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ewellsd@redcross.org" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "120.108.59.206" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1441230787000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 499.48 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CZ" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#e9c943" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 14 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 15 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 15 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 15 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Norma" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Wilson" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "nwilsone@google.com.br" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "18.246.76.220" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1443319848000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : -65.19 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "SE" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#645119" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 15 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 16 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 16 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 16 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Joan" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Watkins" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jwatkinsf@yolasite.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "240.27.33.114" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1459384154000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 264.23 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "PH" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 16 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 17 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 17 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 17 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Gerald" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Hamilton" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ghamiltong@fc2.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "182.75.62.95" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1455114575000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 309.26 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ID" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 17 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 18 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 18 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 18 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Paula" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Taylor" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ptaylorh@wikispaces.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "245.74.203.0" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1462936510000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 927.45 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 18 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 19 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 19 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 19 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Carolyn" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Burns" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "cburnsi@marketwatch.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "180.243.11.10" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1456685363000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 752.76 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "NL" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 19 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "BlankLines.csv" + }, + "sourceOffset" : { + "offset" : 20 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 20 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 20 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Robin" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Bennett" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "rbennettj@cdc.gov" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "169.77.92.179" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1455498404000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 143.30 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ID" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#506128" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "BlankLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/BlankLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1968 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 20 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + } ] +} \ No newline at end of file diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/DataHasMoreFields.json b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/DataHasMoreFields.json index a14bdf5..a47b784 100644 --- a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/DataHasMoreFields.json +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/DataHasMoreFields.json @@ -73,7 +73,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 2 + "offset" : 1 }, "topic" : "testing", "keySchema" : { @@ -323,7 +323,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 2 + "storage" : 1 }, { "name" : "file.last.modified", "schema" : { @@ -339,7 +339,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 3 + "offset" : 2 }, "topic" : "testing", "keySchema" : { @@ -589,7 +589,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 3 + "storage" : 2 }, { "name" : "file.last.modified", "schema" : { @@ -605,7 +605,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 4 + "offset" : 3 }, "topic" : "testing", "keySchema" : { @@ -853,7 +853,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 4 + "storage" : 3 }, { "name" : "file.last.modified", "schema" : { @@ -869,7 +869,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 5 + "offset" : 4 }, "topic" : "testing", "keySchema" : { @@ -1119,7 +1119,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 5 + "storage" : 4 }, { "name" : "file.last.modified", "schema" : { @@ -1135,7 +1135,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 6 + "offset" : 5 }, "topic" : "testing", "keySchema" : { @@ -1384,7 +1384,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 6 + "storage" : 5 }, { "name" : "file.last.modified", "schema" : { @@ -1400,7 +1400,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 7 + "offset" : 6 }, "topic" : "testing", "keySchema" : { @@ -1649,7 +1649,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 7 + "storage" : 6 }, { "name" : "file.last.modified", "schema" : { @@ -1665,7 +1665,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 8 + "offset" : 7 }, "topic" : "testing", "keySchema" : { @@ -1915,7 +1915,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 8 + "storage" : 7 }, { "name" : "file.last.modified", "schema" : { @@ -1931,7 +1931,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 9 + "offset" : 8 }, "topic" : "testing", "keySchema" : { @@ -2180,7 +2180,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 9 + "storage" : 8 }, { "name" : "file.last.modified", "schema" : { @@ -2196,7 +2196,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 10 + "offset" : 9 }, "topic" : "testing", "keySchema" : { @@ -2445,7 +2445,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 10 + "storage" : 9 }, { "name" : "file.last.modified", "schema" : { @@ -2461,7 +2461,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 11 + "offset" : 10 }, "topic" : "testing", "keySchema" : { @@ -2711,7 +2711,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 11 + "storage" : 10 }, { "name" : "file.last.modified", "schema" : { @@ -2727,7 +2727,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 12 + "offset" : 11 }, "topic" : "testing", "keySchema" : { @@ -2977,7 +2977,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 12 + "storage" : 11 }, { "name" : "file.last.modified", "schema" : { @@ -2993,7 +2993,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 13 + "offset" : 12 }, "topic" : "testing", "keySchema" : { @@ -3243,7 +3243,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 13 + "storage" : 12 }, { "name" : "file.last.modified", "schema" : { @@ -3259,7 +3259,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 14 + "offset" : 13 }, "topic" : "testing", "keySchema" : { @@ -3508,7 +3508,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 14 + "storage" : 13 }, { "name" : "file.last.modified", "schema" : { @@ -3524,7 +3524,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 15 + "offset" : 14 }, "topic" : "testing", "keySchema" : { @@ -3774,7 +3774,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 15 + "storage" : 14 }, { "name" : "file.last.modified", "schema" : { @@ -3790,7 +3790,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 16 + "offset" : 15 }, "topic" : "testing", "keySchema" : { @@ -4040,7 +4040,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 16 + "storage" : 15 }, { "name" : "file.last.modified", "schema" : { @@ -4056,7 +4056,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 17 + "offset" : 16 }, "topic" : "testing", "keySchema" : { @@ -4305,7 +4305,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 17 + "storage" : 16 }, { "name" : "file.last.modified", "schema" : { @@ -4321,7 +4321,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 18 + "offset" : 17 }, "topic" : "testing", "keySchema" : { @@ -4570,7 +4570,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 18 + "storage" : 17 }, { "name" : "file.last.modified", "schema" : { @@ -4586,7 +4586,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 19 + "offset" : 18 }, "topic" : "testing", "keySchema" : { @@ -4835,7 +4835,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 19 + "storage" : 18 }, { "name" : "file.last.modified", "schema" : { @@ -4851,7 +4851,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 20 + "offset" : 19 }, "topic" : "testing", "keySchema" : { @@ -5100,7 +5100,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 20 + "storage" : 19 }, { "name" : "file.last.modified", "schema" : { @@ -5116,7 +5116,7 @@ "fileName" : "DataHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 21 + "offset" : 20 }, "topic" : "testing", "keySchema" : { @@ -5366,7 +5366,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 21 + "storage" : 20 }, { "name" : "file.last.modified", "schema" : { diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.json b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.json index 4929aa3..e98b010 100644 --- a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.json +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FieldsMatch.json @@ -73,7 +73,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 2 + "offset" : 1 }, "topic" : "testing", "keySchema" : { @@ -323,7 +323,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 2 + "storage" : 1 }, { "name" : "file.last.modified", "schema" : { @@ -339,7 +339,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 3 + "offset" : 2 }, "topic" : "testing", "keySchema" : { @@ -589,7 +589,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 3 + "storage" : 2 }, { "name" : "file.last.modified", "schema" : { @@ -605,7 +605,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 4 + "offset" : 3 }, "topic" : "testing", "keySchema" : { @@ -853,7 +853,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 4 + "storage" : 3 }, { "name" : "file.last.modified", "schema" : { @@ -869,7 +869,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 5 + "offset" : 4 }, "topic" : "testing", "keySchema" : { @@ -1119,7 +1119,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 5 + "storage" : 4 }, { "name" : "file.last.modified", "schema" : { @@ -1135,7 +1135,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 6 + "offset" : 5 }, "topic" : "testing", "keySchema" : { @@ -1384,7 +1384,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 6 + "storage" : 5 }, { "name" : "file.last.modified", "schema" : { @@ -1400,7 +1400,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 7 + "offset" : 6 }, "topic" : "testing", "keySchema" : { @@ -1649,7 +1649,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 7 + "storage" : 6 }, { "name" : "file.last.modified", "schema" : { @@ -1665,7 +1665,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 8 + "offset" : 7 }, "topic" : "testing", "keySchema" : { @@ -1915,7 +1915,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 8 + "storage" : 7 }, { "name" : "file.last.modified", "schema" : { @@ -1931,7 +1931,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 9 + "offset" : 8 }, "topic" : "testing", "keySchema" : { @@ -2180,7 +2180,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 9 + "storage" : 8 }, { "name" : "file.last.modified", "schema" : { @@ -2196,7 +2196,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 10 + "offset" : 9 }, "topic" : "testing", "keySchema" : { @@ -2445,7 +2445,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 10 + "storage" : 9 }, { "name" : "file.last.modified", "schema" : { @@ -2461,7 +2461,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 11 + "offset" : 10 }, "topic" : "testing", "keySchema" : { @@ -2711,7 +2711,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 11 + "storage" : 10 }, { "name" : "file.last.modified", "schema" : { @@ -2727,7 +2727,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 12 + "offset" : 11 }, "topic" : "testing", "keySchema" : { @@ -2977,7 +2977,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 12 + "storage" : 11 }, { "name" : "file.last.modified", "schema" : { @@ -2993,7 +2993,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 13 + "offset" : 12 }, "topic" : "testing", "keySchema" : { @@ -3243,7 +3243,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 13 + "storage" : 12 }, { "name" : "file.last.modified", "schema" : { @@ -3259,7 +3259,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 14 + "offset" : 13 }, "topic" : "testing", "keySchema" : { @@ -3508,7 +3508,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 14 + "storage" : 13 }, { "name" : "file.last.modified", "schema" : { @@ -3524,7 +3524,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 15 + "offset" : 14 }, "topic" : "testing", "keySchema" : { @@ -3774,7 +3774,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 15 + "storage" : 14 }, { "name" : "file.last.modified", "schema" : { @@ -3790,7 +3790,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 16 + "offset" : 15 }, "topic" : "testing", "keySchema" : { @@ -4040,7 +4040,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 16 + "storage" : 15 }, { "name" : "file.last.modified", "schema" : { @@ -4056,7 +4056,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 17 + "offset" : 16 }, "topic" : "testing", "keySchema" : { @@ -4305,7 +4305,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 17 + "storage" : 16 }, { "name" : "file.last.modified", "schema" : { @@ -4321,7 +4321,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 18 + "offset" : 17 }, "topic" : "testing", "keySchema" : { @@ -4570,7 +4570,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 18 + "storage" : 17 }, { "name" : "file.last.modified", "schema" : { @@ -4586,7 +4586,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 19 + "offset" : 18 }, "topic" : "testing", "keySchema" : { @@ -4835,7 +4835,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 19 + "storage" : 18 }, { "name" : "file.last.modified", "schema" : { @@ -4851,7 +4851,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 20 + "offset" : 19 }, "topic" : "testing", "keySchema" : { @@ -5100,7 +5100,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 20 + "storage" : 19 }, { "name" : "file.last.modified", "schema" : { @@ -5116,7 +5116,7 @@ "fileName" : "FieldsMatch.csv" }, "sourceOffset" : { - "offset" : 21 + "offset" : 20 }, "topic" : "testing", "keySchema" : { @@ -5366,7 +5366,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 21 + "storage" : 20 }, { "name" : "file.last.modified", "schema" : { diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FileModeFieldFieldsMatch.json b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FileModeFieldFieldsMatch.json index f3b93e2..de76e23 100644 --- a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FileModeFieldFieldsMatch.json +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/FileModeFieldFieldsMatch.json @@ -73,7 +73,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 2 + "offset" : 1 }, "topic" : "testing", "keySchema" : { @@ -323,7 +323,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 2 + "storage" : 1 }, { "name" : "file.last.modified", "schema" : { @@ -339,7 +339,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 3 + "offset" : 2 }, "topic" : "testing", "keySchema" : { @@ -589,7 +589,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 3 + "storage" : 2 }, { "name" : "file.last.modified", "schema" : { @@ -605,7 +605,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 4 + "offset" : 3 }, "topic" : "testing", "keySchema" : { @@ -854,7 +854,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 4 + "storage" : 3 }, { "name" : "file.last.modified", "schema" : { @@ -870,7 +870,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 5 + "offset" : 4 }, "topic" : "testing", "keySchema" : { @@ -1120,7 +1120,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 5 + "storage" : 4 }, { "name" : "file.last.modified", "schema" : { @@ -1136,7 +1136,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 6 + "offset" : 5 }, "topic" : "testing", "keySchema" : { @@ -1385,7 +1385,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 6 + "storage" : 5 }, { "name" : "file.last.modified", "schema" : { @@ -1401,7 +1401,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 7 + "offset" : 6 }, "topic" : "testing", "keySchema" : { @@ -1650,7 +1650,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 7 + "storage" : 6 }, { "name" : "file.last.modified", "schema" : { @@ -1666,7 +1666,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 8 + "offset" : 7 }, "topic" : "testing", "keySchema" : { @@ -1916,7 +1916,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 8 + "storage" : 7 }, { "name" : "file.last.modified", "schema" : { @@ -1932,7 +1932,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 9 + "offset" : 8 }, "topic" : "testing", "keySchema" : { @@ -2181,7 +2181,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 9 + "storage" : 8 }, { "name" : "file.last.modified", "schema" : { @@ -2197,7 +2197,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 10 + "offset" : 9 }, "topic" : "testing", "keySchema" : { @@ -2446,7 +2446,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 10 + "storage" : 9 }, { "name" : "file.last.modified", "schema" : { @@ -2462,7 +2462,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 11 + "offset" : 10 }, "topic" : "testing", "keySchema" : { @@ -2712,7 +2712,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 11 + "storage" : 10 }, { "name" : "file.last.modified", "schema" : { @@ -2728,7 +2728,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 12 + "offset" : 11 }, "topic" : "testing", "keySchema" : { @@ -2978,7 +2978,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 12 + "storage" : 11 }, { "name" : "file.last.modified", "schema" : { @@ -2994,7 +2994,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 13 + "offset" : 12 }, "topic" : "testing", "keySchema" : { @@ -3244,7 +3244,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 13 + "storage" : 12 }, { "name" : "file.last.modified", "schema" : { @@ -3260,7 +3260,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 14 + "offset" : 13 }, "topic" : "testing", "keySchema" : { @@ -3509,7 +3509,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 14 + "storage" : 13 }, { "name" : "file.last.modified", "schema" : { @@ -3525,7 +3525,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 15 + "offset" : 14 }, "topic" : "testing", "keySchema" : { @@ -3775,7 +3775,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 15 + "storage" : 14 }, { "name" : "file.last.modified", "schema" : { @@ -3791,7 +3791,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 16 + "offset" : 15 }, "topic" : "testing", "keySchema" : { @@ -4041,7 +4041,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 16 + "storage" : 15 }, { "name" : "file.last.modified", "schema" : { @@ -4057,7 +4057,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 17 + "offset" : 16 }, "topic" : "testing", "keySchema" : { @@ -4306,7 +4306,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 17 + "storage" : 16 }, { "name" : "file.last.modified", "schema" : { @@ -4322,7 +4322,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 18 + "offset" : 17 }, "topic" : "testing", "keySchema" : { @@ -4571,7 +4571,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 18 + "storage" : 17 }, { "name" : "file.last.modified", "schema" : { @@ -4587,7 +4587,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 19 + "offset" : 18 }, "topic" : "testing", "keySchema" : { @@ -4836,7 +4836,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 19 + "storage" : 18 }, { "name" : "file.last.modified", "schema" : { @@ -4852,7 +4852,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 20 + "offset" : 19 }, "topic" : "testing", "keySchema" : { @@ -5101,7 +5101,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 20 + "storage" : 19 }, { "name" : "file.last.modified", "schema" : { @@ -5117,7 +5117,7 @@ "fileName" : "FileModeFieldFieldsMatch.csv" }, "sourceOffset" : { - "offset" : 21 + "offset" : 20 }, "topic" : "testing", "keySchema" : { @@ -5367,7 +5367,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 21 + "storage" : 20 }, { "name" : "file.last.modified", "schema" : { diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SchemaHasMoreFields.json b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SchemaHasMoreFields.json index fe6bdf2..b30e151 100644 --- a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SchemaHasMoreFields.json +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SchemaHasMoreFields.json @@ -73,7 +73,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 2 + "offset" : 1 }, "topic" : "testing", "keySchema" : { @@ -322,7 +322,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 2 + "storage" : 1 }, { "name" : "file.last.modified", "schema" : { @@ -338,7 +338,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 3 + "offset" : 2 }, "topic" : "testing", "keySchema" : { @@ -587,7 +587,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 3 + "storage" : 2 }, { "name" : "file.last.modified", "schema" : { @@ -603,7 +603,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 4 + "offset" : 3 }, "topic" : "testing", "keySchema" : { @@ -851,7 +851,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 4 + "storage" : 3 }, { "name" : "file.last.modified", "schema" : { @@ -867,7 +867,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 5 + "offset" : 4 }, "topic" : "testing", "keySchema" : { @@ -1116,7 +1116,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 5 + "storage" : 4 }, { "name" : "file.last.modified", "schema" : { @@ -1132,7 +1132,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 6 + "offset" : 5 }, "topic" : "testing", "keySchema" : { @@ -1381,7 +1381,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 6 + "storage" : 5 }, { "name" : "file.last.modified", "schema" : { @@ -1397,7 +1397,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 7 + "offset" : 6 }, "topic" : "testing", "keySchema" : { @@ -1646,7 +1646,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 7 + "storage" : 6 }, { "name" : "file.last.modified", "schema" : { @@ -1662,7 +1662,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 8 + "offset" : 7 }, "topic" : "testing", "keySchema" : { @@ -1911,7 +1911,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 8 + "storage" : 7 }, { "name" : "file.last.modified", "schema" : { @@ -1927,7 +1927,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 9 + "offset" : 8 }, "topic" : "testing", "keySchema" : { @@ -2176,7 +2176,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 9 + "storage" : 8 }, { "name" : "file.last.modified", "schema" : { @@ -2192,7 +2192,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 10 + "offset" : 9 }, "topic" : "testing", "keySchema" : { @@ -2441,7 +2441,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 10 + "storage" : 9 }, { "name" : "file.last.modified", "schema" : { @@ -2457,7 +2457,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 11 + "offset" : 10 }, "topic" : "testing", "keySchema" : { @@ -2706,7 +2706,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 11 + "storage" : 10 }, { "name" : "file.last.modified", "schema" : { @@ -2722,7 +2722,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 12 + "offset" : 11 }, "topic" : "testing", "keySchema" : { @@ -2971,7 +2971,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 12 + "storage" : 11 }, { "name" : "file.last.modified", "schema" : { @@ -2987,7 +2987,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 13 + "offset" : 12 }, "topic" : "testing", "keySchema" : { @@ -3236,7 +3236,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 13 + "storage" : 12 }, { "name" : "file.last.modified", "schema" : { @@ -3252,7 +3252,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 14 + "offset" : 13 }, "topic" : "testing", "keySchema" : { @@ -3501,7 +3501,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 14 + "storage" : 13 }, { "name" : "file.last.modified", "schema" : { @@ -3517,7 +3517,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 15 + "offset" : 14 }, "topic" : "testing", "keySchema" : { @@ -3766,7 +3766,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 15 + "storage" : 14 }, { "name" : "file.last.modified", "schema" : { @@ -3782,7 +3782,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 16 + "offset" : 15 }, "topic" : "testing", "keySchema" : { @@ -4031,7 +4031,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 16 + "storage" : 15 }, { "name" : "file.last.modified", "schema" : { @@ -4047,7 +4047,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 17 + "offset" : 16 }, "topic" : "testing", "keySchema" : { @@ -4296,7 +4296,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 17 + "storage" : 16 }, { "name" : "file.last.modified", "schema" : { @@ -4312,7 +4312,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 18 + "offset" : 17 }, "topic" : "testing", "keySchema" : { @@ -4561,7 +4561,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 18 + "storage" : 17 }, { "name" : "file.last.modified", "schema" : { @@ -4577,7 +4577,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 19 + "offset" : 18 }, "topic" : "testing", "keySchema" : { @@ -4826,7 +4826,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 19 + "storage" : 18 }, { "name" : "file.last.modified", "schema" : { @@ -4842,7 +4842,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 20 + "offset" : 19 }, "topic" : "testing", "keySchema" : { @@ -5091,7 +5091,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 20 + "storage" : 19 }, { "name" : "file.last.modified", "schema" : { @@ -5107,7 +5107,7 @@ "fileName" : "SchemaHasMoreFields.csv" }, "sourceOffset" : { - "offset" : 21 + "offset" : 20 }, "topic" : "testing", "keySchema" : { @@ -5356,7 +5356,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 21 + "storage" : 20 }, { "name" : "file.last.modified", "schema" : { diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SourceOffset.json b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SourceOffset.json index a708bb7..889f682 100644 --- a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SourceOffset.json +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/SourceOffset.json @@ -75,7 +75,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 11 + "offset" : 10 }, "topic" : "testing", "keySchema" : { @@ -325,7 +325,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 11 + "storage" : 10 }, { "name" : "file.last.modified", "schema" : { @@ -341,7 +341,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 12 + "offset" : 11 }, "topic" : "testing", "keySchema" : { @@ -591,7 +591,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 12 + "storage" : 11 }, { "name" : "file.last.modified", "schema" : { @@ -607,7 +607,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 13 + "offset" : 12 }, "topic" : "testing", "keySchema" : { @@ -857,7 +857,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 13 + "storage" : 12 }, { "name" : "file.last.modified", "schema" : { @@ -873,7 +873,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 14 + "offset" : 13 }, "topic" : "testing", "keySchema" : { @@ -1122,7 +1122,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 14 + "storage" : 13 }, { "name" : "file.last.modified", "schema" : { @@ -1138,7 +1138,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 15 + "offset" : 14 }, "topic" : "testing", "keySchema" : { @@ -1388,7 +1388,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 15 + "storage" : 14 }, { "name" : "file.last.modified", "schema" : { @@ -1404,7 +1404,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 16 + "offset" : 15 }, "topic" : "testing", "keySchema" : { @@ -1654,7 +1654,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 16 + "storage" : 15 }, { "name" : "file.last.modified", "schema" : { @@ -1670,7 +1670,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 17 + "offset" : 16 }, "topic" : "testing", "keySchema" : { @@ -1919,7 +1919,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 17 + "storage" : 16 }, { "name" : "file.last.modified", "schema" : { @@ -1935,7 +1935,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 18 + "offset" : 17 }, "topic" : "testing", "keySchema" : { @@ -2184,7 +2184,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 18 + "storage" : 17 }, { "name" : "file.last.modified", "schema" : { @@ -2200,7 +2200,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 19 + "offset" : 18 }, "topic" : "testing", "keySchema" : { @@ -2449,7 +2449,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 19 + "storage" : 18 }, { "name" : "file.last.modified", "schema" : { @@ -2465,7 +2465,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 20 + "offset" : 19 }, "topic" : "testing", "keySchema" : { @@ -2714,7 +2714,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 20 + "storage" : 19 }, { "name" : "file.last.modified", "schema" : { @@ -2730,7 +2730,7 @@ "fileName" : "SourceOffset.csv" }, "sourceOffset" : { - "offset" : 21 + "offset" : 20 }, "topic" : "testing", "keySchema" : { @@ -2980,7 +2980,7 @@ "type" : "INT64", "isOptional" : false }, - "storage" : 21 + "storage" : 20 }, { "name" : "file.last.modified", "schema" : { diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithHeaderSkipLines.data b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithHeaderSkipLines.data new file mode 100644 index 0000000..d3e9ae2 --- /dev/null +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithHeaderSkipLines.data @@ -0,0 +1,22 @@ +#skip this line +id,first_name,last_name,email,gender,ip_address,last_login,account_balance,country,favorite_color +1,Jack,Garcia,jgarcia0@shop-pro.jp,Male,196.56.44.185,2015-09-30T15:29:03Z,347.77,IT,#4a2313 +2,John,Kim,jkim1@miibeian.gov.cn,Male,53.19.132.185,2015-11-14T10:34:09Z,251.24,CZ,#3e56cf +3,Ashley,Austin,aaustin2@hatena.ne.jp,Female,21.164.37.9,,819.47,CN, +4,Jonathan,Mcdonald,jmcdonald3@amazon.co.uk,Male,188.172.42.140,2015-12-28T14:37:01Z,868.38,ID,#1b1414 +5,Helen,Lane,hlane4@trellian.com,Female,159.171.138.190,2016-06-30T18:41:18Z,398.97,TN, +6,Scott,Lopez,slopez5@google.co.jp,Male,86.194.226.35,2015-08-13T02:13:51Z,322.99,BR, +7,Christine,Franklin,cfranklin6@reuters.com,Female,248.173.207.64,2015-12-22T11:29:57Z,301.26,PH,#1d5e9d +8,Helen,Andrews,handrews7@histats.com,Female,83.160.63.181,2016-03-06T11:41:10Z,217.96,CU, +9,Stephanie,Gordon,sgordon8@goodreads.com,Female,193.143.42.212,2015-10-27T22:07:24Z,495.80,CN, +10,Shirley,Andrews,sandrews9@flickr.com,Female,99.113.183.206,2015-11-07T11:12:52Z,157.75,BR,#fc1da9 +11,Joshua,Reid,jreida@wikia.com,Male,197.96.118.164,2015-08-22T13:16:18Z,431.80,CO,#6e3e36 +12,Frances,Parker,fparkerb@engadget.com,Female,226.237.57.25,2015-10-18T01:50:15Z,188.21,BR,#73e909 +13,Sharon,Lawson,slawsonc@bravesites.com,Female,198.189.134.106,2016-01-14T17:51:09Z,206.73,VN, +14,Elizabeth,Wells,ewellsd@redcross.org,Female,120.108.59.206,2015-09-02T21:53:07Z,499.48,CZ,#e9c943 +15,Norma,Wilson,nwilsone@google.com.br,Female,18.246.76.220,2015-09-27T02:10:48Z,-65.19,SE,#645119 +16,Joan,Watkins,jwatkinsf@yolasite.com,Female,240.27.33.114,2016-03-31T00:29:14Z,264.23,PH, +17,Gerald,Hamilton,ghamiltong@fc2.com,Male,182.75.62.95,2016-02-10T14:29:35Z,309.26,ID, +18,Paula,Taylor,ptaylorh@wikispaces.com,Female,245.74.203.0,2016-05-11T03:15:10Z,927.45,CN, +19,Carolyn,Burns,cburnsi@marketwatch.com,Female,180.243.11.10,2016-02-28T18:49:23Z,752.76,NL, +20,Robin,Bennett,rbennettj@cdc.gov,Female,169.77.92.179,2016-02-15T01:06:44Z,143.30,ID,#506128 \ No newline at end of file diff --git a/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithHeaderSkipLines.json b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithHeaderSkipLines.json new file mode 100644 index 0000000..95ae95f --- /dev/null +++ b/src/test/resources/com/github/jcustenborder/kafka/connect/spooldir/csv/WithHeaderSkipLines.json @@ -0,0 +1,5383 @@ +{ + "settings" : { + "csv.first.row.as.header" : "true", + "csv.skip.lines": "1", + "schema.generation.enabled": "true" + }, + "offset" : { }, + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "expected" : [ { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 1 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Jack" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Garcia" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jgarcia0@shop-pro.jp" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "196.56.44.185" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1443626943000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 347.77 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "IT" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#4a2313" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 2 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 2 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 2 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "John" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Kim" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jkim1@miibeian.gov.cn" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "53.19.132.185" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1447497249000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 251.24 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CZ" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#3e56cf" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 2 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 3 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 3 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 3 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Ashley" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Austin" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "aaustin2@hatena.ne.jp" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "21.164.37.9" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + } + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 819.47 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 3 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 4 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 4 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 4 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Jonathan" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Mcdonald" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jmcdonald3@amazon.co.uk" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "188.172.42.140" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1451313421000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 868.38 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ID" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#1b1414" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 4 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 5 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 5 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 5 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Helen" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Lane" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "hlane4@trellian.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "159.171.138.190" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1467312078000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 398.97 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "TN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 5 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 6 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 6 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 6 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Scott" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Lopez" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "slopez5@google.co.jp" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "86.194.226.35" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1439432031000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 322.99 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "BR" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 6 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 7 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 7 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 7 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Christine" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Franklin" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "cfranklin6@reuters.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "248.173.207.64" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1450783797000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 301.26 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "PH" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#1d5e9d" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 7 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 8 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 8 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 8 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Helen" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Andrews" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "handrews7@histats.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "83.160.63.181" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1457264470000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 217.96 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CU" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 8 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 9 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 9 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 9 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Stephanie" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Gordon" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "sgordon8@goodreads.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "193.143.42.212" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1445983644000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 495.80 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 9 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 10 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 10 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 10 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Shirley" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Andrews" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "sandrews9@flickr.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "99.113.183.206" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1446894772000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 157.75 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "BR" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#fc1da9" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 10 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 11 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 11 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 11 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Joshua" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Reid" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jreida@wikia.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "197.96.118.164" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1440249378000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 431.80 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CO" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#6e3e36" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 11 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 12 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 12 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 12 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Frances" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Parker" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "fparkerb@engadget.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "226.237.57.25" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1445133015000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 188.21 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "BR" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#73e909" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 12 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 13 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 13 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 13 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Sharon" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Lawson" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "slawsonc@bravesites.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "198.189.134.106" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1452793869000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 206.73 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "VN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 13 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 14 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 14 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 14 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Elizabeth" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Wells" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ewellsd@redcross.org" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "120.108.59.206" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1441230787000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 499.48 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CZ" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#e9c943" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 14 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 15 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 15 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 15 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Norma" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Wilson" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "nwilsone@google.com.br" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "18.246.76.220" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1443319848000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : -65.19 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "SE" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#645119" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 15 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 16 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 16 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 16 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Joan" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Watkins" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "jwatkinsf@yolasite.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "240.27.33.114" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1459384154000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 264.23 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "PH" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 16 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 17 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 17 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 17 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Gerald" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Hamilton" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ghamiltong@fc2.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Male" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "182.75.62.95" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1455114575000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 309.26 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ID" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 17 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 18 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 18 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 18 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Paula" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Taylor" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ptaylorh@wikispaces.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "245.74.203.0" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1462936510000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 927.45 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "CN" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 18 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 19 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 19 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 19 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Carolyn" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Burns" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "cburnsi@marketwatch.com" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "180.243.11.10" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1456685363000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 752.76 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "NL" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + } + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 19 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + }, { + "sourcePartition" : { + "fileName" : "WithHeaderSkipLines.csv" + }, + "sourceOffset" : { + "offset" : 20 + }, + "topic" : "testing", + "keySchema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "key" : { + "schema" : { + "name" : "com.example.users.UserKey", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 20 + } ] + }, + "valueSchema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "value" : { + "schema" : { + "name" : "com.example.users.User", + "type" : "STRUCT", + "isOptional" : false, + "fieldSchemas" : { + "id" : { + "type" : "INT64", + "isOptional" : false + }, + "first_name" : { + "type" : "STRING", + "isOptional" : true + }, + "last_name" : { + "type" : "STRING", + "isOptional" : true + }, + "email" : { + "type" : "STRING", + "isOptional" : true + }, + "gender" : { + "type" : "STRING", + "isOptional" : true + }, + "ip_address" : { + "type" : "STRING", + "isOptional" : true + }, + "last_login" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "account_balance" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "country" : { + "type" : "STRING", + "isOptional" : true + }, + "favorite_color" : { + "type" : "STRING", + "isOptional" : true + } + } + }, + "fieldValues" : [ { + "name" : "id", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 20 + }, { + "name" : "first_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Robin" + }, { + "name" : "last_name", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Bennett" + }, { + "name" : "email", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "rbennettj@cdc.gov" + }, { + "name" : "gender", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "Female" + }, { + "name" : "ip_address", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "169.77.92.179" + }, { + "name" : "last_login", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : true + }, + "storage" : 1455498404000 + }, { + "name" : "account_balance", + "schema" : { + "name" : "org.apache.kafka.connect.data.Decimal", + "type" : "BYTES", + "version" : 1, + "parameters" : { + "scale" : "2" + }, + "isOptional" : true + }, + "storage" : 143.30 + }, { + "name" : "country", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "ID" + }, { + "name" : "favorite_color", + "schema" : { + "type" : "STRING", + "isOptional" : true + }, + "storage" : "#506128" + } ] + }, + "headers" : [ { + "name" : "file.name", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "WithHeaderSkipLines.csv" + }, { + "name" : "file.path", + "schema" : { + "type" : "STRING", + "isOptional" : false + }, + "storage" : "/var/folders/fc/jqgphp3s5l9087p4v7pdxh040000gn/T/1559925244697-0/input/WithHeaderSkipLines.csv" + }, { + "name" : "file.length", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 1870 + }, { + "name" : "file.offset", + "schema" : { + "type" : "INT64", + "isOptional" : false + }, + "storage" : 20 + }, { + "name" : "file.last.modified", + "schema" : { + "name" : "org.apache.kafka.connect.data.Timestamp", + "type" : "INT64", + "version" : 1, + "isOptional" : false + }, + "storage" : 1559925244000 + } ] + } ] +} \ No newline at end of file