SNOW-1332640 support reading of structured types

lib/connection/result/column.js

@@ -10,7 +10,10 @@
 const SfTimestamp = require('./sf_timestamp');
 const DataTypes = require('./data_types');
 const SqlTypes = require('./data_types').SqlTypes;
+const dateTimeFormatConverter = require('./datetime_format_converter');
 const bigInt = require('big-integer');
+const moment = require('moment');
+const momentTimezone = require('moment-timezone');
 
 /**
  * Creates a new Column.
@@ -28,6 +31,7 @@
   const scale = options.scale;
   const type = options.type;
   const precision = options.precision;
+  const fieldsMetadata = options.fields;
 
   /**
    * Returns the name of this column.
@@ -104,9 +108,9 @@
   this.isTimestampLtz = createFnIsColumnOfType(type, SqlTypes.isTimestampLtz, SqlTypes);
   this.isTimestampNtz = createFnIsColumnOfType(type, SqlTypes.isTimestampNtz, SqlTypes);
   this.isTimestampTz = createFnIsColumnOfType(type, SqlTypes.isTimestampTz, SqlTypes);
-  this.isVariant = createFnIsColumnOfType(type, SqlTypes.isVariant, SqlTypes);
-  this.isObject = createFnIsColumnOfType(type, SqlTypes.isObject, SqlTypes);
-  this.isArray = createFnIsColumnOfType(type, SqlTypes.isArray, SqlTypes);
+  this.isVariant = createFnIsColumnOfType(type, () => SqlTypes.isVariant(type, fieldsMetadata), SqlTypes);
+  this.isObject = createFnIsColumnOfType(type, () => SqlTypes.isObject(type, fieldsMetadata), SqlTypes);
+  this.isArray = createFnIsColumnOfType(type, () => SqlTypes.isArray(type, fieldsMetadata), SqlTypes);
 
   let convert;
   let toString;
@@ -168,6 +172,9 @@
     } else if (this.isVariant()) {
       convert = convertRawVariant;
       toString = toStringFromVariant;
+    } else if (this.isObject()) {
+      convert = convertRawObject;
+      toString = toStringFromVariant;
     } else {
       // column is of type string, so leave value as is
       convert = noop;
@@ -183,7 +190,8 @@
       toString: toString,
       format: format,
       resultVersion: resultVersion,
-      statementParameters: statementParameters
+      statementParameters: statementParameters,
+      fieldsMetadata: fieldsMetadata
     };
 
   /**
@@ -268,14 +276,107 @@
 function convertRawBoolean(rawColumnValue) {
   let ret;
 
-  if ((rawColumnValue === '1') || (rawColumnValue === 'TRUE')) {
+  if (rawColumnValue  === true || (rawColumnValue === '1') || (rawColumnValue === 'TRUE')) {
     ret = true;
-  } else if ((rawColumnValue === '0') || (rawColumnValue === 'FALSE')) {
+  } else if (rawColumnValue === false || (rawColumnValue === '0') || (rawColumnValue === 'FALSE')) {
     ret = false;
+  } else {
+    throw new Error(`Value could not be converted to boolean: ${rawColumnValue}`);
   }
-
   return ret;
 }
+/**
+ * Converts a raw column value of structured type object to javascript Object
+ *
+ * @param {String} rawColumnValue
+ *
+ * @returns {Object}
+ */
+function convertRawObject(rawColumnValue, column, context) {
+  const fieldsByName =  context.fieldsMetadata.reduce(function (map, obj) {
+    map[obj.name] = obj;
+    return map;
+  }, {});
+  const json = JSON.parse(rawColumnValue);
+  const result = {};
+
+  Object.keys(json).forEach(function (key) {
+    result[key] = mapStructuredTypeFieldValue(json[key], fieldsByName[key], context, column);
+  });
+
+  return result;
+}
+
+function mapStructuredTypeFieldValue(columnValue, columnMetadata, context, column) {
+  const formatLtz = context.statementParameters['TIMESTAMP_LTZ_OUTPUT_FORMAT'] || context.statementParameters['TIMESTAMP_OUTPUT_FORMAT'];
+  const formatTz = context.statementParameters['TIMESTAMP_TZ_OUTPUT_FORMAT'] || context.statementParameters['TIMESTAMP_OUTPUT_FORMAT'];
+  const formatNtz = context.statementParameters['TIMESTAMP_NTZ_OUTPUT_FORMAT'];
+  let value;
+  switch (columnMetadata.type) {
+  case 'text':
+    value =  columnValue;
+    break;
+  case 'real':
+    value =  toValueFromNumber(convertRawNumber(columnValue));
+    break;
+  case 'fixed':
+    value =  toValueFromNumber(convertRawNumber(columnValue));
+    break;
+  case 'boolean':
+    value =  convertRawBoolean(columnValue);
+    break;
+  case 'timestamp_ltz':
+    value = convertTimestampTzString(columnValue, formatLtz, context.statementParameters['TIMEZONE'], column.getScale()).toSfDate();
+    break;
+  case 'timestamp_ntz':
+    value = convertTimestampNtzString(columnValue, formatNtz, moment.tz.zone('UTC'), column.getScale()).toSfDate();
+    break;
+  case 'timestamp_tz':
+    value = convertTimestampTzString(columnValue, formatTz, context.statementParameters['TIMEZONE'], column.getScale()).toSfDate();
+    break;
+  case 'date':
+    context.format = context.statementParameters['DATE_OUTPUT_FORMAT'];
+    value = convertTimestampTzString(columnValue, context.format, context.statementParameters['TIMEZONE'], column.getScale()).toSfDate();
+    break;
+  case 'time':
+    context.format = context.statementParameters['TIME_OUTPUT_FORMAT'];
+    value = convertTimeString(columnValue, context.format, moment.tz.zone('UTC'), column.getScale()).toSfTime();
+    break;
+  case 'binary':
+    context.format = context.statementParameters['BINARY_OUTPUT_FORMAT'];
+    value =  convertRawBinary(columnValue, column, context).toJSON().data;
+    break;
+  case 'object':
+    value =  convertRawObject(columnValue, column, context).toJSON().data;
+    break;
+  default:
+    Logger.getInstance().info(`Column type not supported: ${columnMetadata.type}`);
+    throw new Error(`Column type not supported: ${columnMetadata.type}`);
+  }
+  return value;
+}
+
+const convertTimestampTzString = function (stringValue, formatSql, timezone, scale) {
+  const formatMoment = dateTimeFormatConverter.convertSnowflakeFormatToMomentFormat(formatSql, scale);
+  const epochSeconds = momentTimezone(stringValue, formatMoment).unix();
+  return new SfTimestamp(epochSeconds, 0, scale, timezone, formatSql);
+};
+
+const convertTimestampNtzString = function (stringValue, formatSql, timezone, scale) {
+  const formatMoment = dateTimeFormatConverter.convertSnowflakeFormatToMomentFormat(formatSql, scale);
+  const epochSeconds = momentTimezone.utc(stringValue, formatMoment).unix();
+  return new SfTimestamp(epochSeconds, 0, scale, timezone, formatSql);
+};
+
+
+const convertTimeString = function (stringValue, formatSql, timezone, scale) {
+  const formatMoment = dateTimeFormatConverter.convertSnowflakeFormatToMomentFormat(formatSql, scale);
+  const moment = momentTimezone(stringValue, formatMoment);
+  const epochSeconds = moment.hours() * 3600 + moment.minutes() * 60 + moment.seconds();
+  const time =  new SfTimestamp(epochSeconds, 0, scale, timezone, formatSql);
+  time._valueAsString = stringValue;
+  return time;
+};
 
 /**
  * Converts a raw column value of type Date to a Snowflake Date.

lib/connection/result/data_types.js

@@ -143,9 +143,9 @@
      *
      * @returns {Boolean}
      */
-    isVariant: function (sqlType) {
+    isVariant: function (sqlType, fieldsMetadata) {
       return (sqlType === this.values.VARIANT) ||
-        (sqlType === this.values.OBJECT) ||
+        (sqlType === this.values.OBJECT && fieldsMetadata == null) ||
         (sqlType === this.values.ARRAY);
     },
 
@@ -156,8 +156,8 @@
      *
      * @returns {Boolean}
      */
-    isObject: function (sqlType) {
-      return (sqlType === this.values.OBJECT);
+    isObject: function (sqlType, fieldsMetadata) {
+      return (sqlType === this.values.OBJECT && fieldsMetadata != null);
     },
 
     /**
@@ -167,8 +167,8 @@
      *
      * @returns {Boolean}
      */
-    isArray: function (sqlType) {
-      return (sqlType === this.values.ARRAY);
+    isArray: function (sqlType, fieldsMetadata) {
+      return (sqlType === this.values.ARRAY && fieldsMetadata != null);
     }
   };
 
@@ -181,7 +181,10 @@
         NUMBER: 'NUMBER',
         DATE: 'DATE',
         JSON: 'JSON',
-        BUFFER: 'BUFFER'
+        BUFFER: 'BUFFER',
+        OBJECT: 'OBJECT',
+        ARRAY: 'ARRAY',
+        MAP: 'MAP'
       },
 
     /**
@@ -246,8 +249,9 @@
 MAP_SQL_TO_NATIVE[sqlTypeValues.TIMESTAMP_NTZ] = nativeTypeValues.DATE;
 MAP_SQL_TO_NATIVE[sqlTypeValues.TIMESTAMP_TZ] = nativeTypeValues.DATE;
 MAP_SQL_TO_NATIVE[sqlTypeValues.VARIANT] = nativeTypeValues.JSON;
-MAP_SQL_TO_NATIVE[sqlTypeValues.OBJECT] = nativeTypeValues.JSON;
-MAP_SQL_TO_NATIVE[sqlTypeValues.ARRAY] = nativeTypeValues.JSON;
+MAP_SQL_TO_NATIVE[sqlTypeValues.OBJECT] = nativeTypeValues.OBJECT;
+MAP_SQL_TO_NATIVE[sqlTypeValues.ARRAY] = nativeTypeValues.ARRAY;
+MAP_SQL_TO_NATIVE[sqlTypeValues.MAP] = nativeTypeValues.MAP;
 
 exports.SqlTypes = sqlTypes;
 exports.NativeTypes = nativeTypes;

lib/connection/result/datetime_format_converter.js

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+function formatTagsMap() {
+  return [
+    // proper mappings
+    ['YYYY', 'YYYY'],
+    ['YY', 'YY'],
+    ['MM', 'MM'],
+    ['MON', 'MMM'],
+    ['DD', 'DD'],
+    ['DY', 'ddd'],
+    ['HH24', 'HH'],
+    ['HH12', 'hh'],
+    ['HH', 'HH'],
+    ['AM', 'A'],
+    ['PM', 'A'],
+    ['MI', 'mm'],
+    ['SS', 'ss'],
+    ['TZH:TZM', 'Z'],
+    ['TZHTZM', 'ZZ'],
+
+    // special code needed
+    ['TZH', ''],
+    ['TZM', ''],
+    ['FF', '']
+  ];
+}
+
+function convertSnowflakeFormatToMomentFormat(formatSql, scale) {
+  const tags = formatTagsMap();
+
+  // get an upper-case version of the input sql format
+  const formatSqlUpper = formatSql.toUpperCase();
+  let moment;
+
+  // iterate over the format string
+  const length = formatSql.length;
+  let formatMoment = '';
+  for (let pos = 0; pos < length;) {
+    let tag = null;
+    let out = null;
+
+    // at each position, check if there's a tag at that position; if so, use
+    // 'out' as the replacement
+    for (let index = 0; index < tags.length; index++) {
+      if (formatSqlUpper.substr(pos).indexOf(tags[index][0]) === 0) {
+        tag = tags[index][0];
+        out = tags[index][1];
+        break;
+      }
+    }
+
+    // if we didn't find a match, just insert the character after escaping it
+    // (by wrapping it in square brackets)
+    if (out === null) {
+      formatMoment += formatSql[pos];
+      pos++;
+    } else {
+      // we found one of our special tags
+      if (out === '') {
+        if (tag === 'TZH') {
+          // format the moment to get the timezone string and extract the
+          // hours; for example, '-0700' will be converted to '-07'
+          out = moment.format('ZZ').substr(0, 3);
+        } else if (tag === 'TZM') {
+          // format the moment to get the timezone string and extract the
+          // minutes; for example, '-0700' will be converted to '00
+          out = moment.format('ZZ').substr(3);
+        } else if (tag === 'FF') {
+          // if 'FF' is followed by a digit, use the digit as the scale
+          let digit = null;
+          if (pos + tag.length < length) {
+            const matches = formatSql[pos + tag.length].match(/[0-9]/);
+            if (matches) {
+              digit = matches[0];
+            }
+          }
+          if (digit !== null) {
+            pos++; // skip the digit as well
+          }
+
+          // if we need to include fractional seconds
+          if (scale > 0) {
+            // divide the nanoSeconds to get the requested number of
+            // meaningful digits
+            // pad with the appropriate number of leading zeros
+            out = (new Array(9).join('S')).substr(-scale);
+          }
+        }
+      }
+
+      // append the 'out' text to the moment format and update the position
+      formatMoment += out;
+      pos += tag.length;
+    }
+  }
+  return formatMoment;
+}
+
+module.exports.formatTagsMap = formatTagsMap;
+module.exports.convertSnowflakeFormatToMomentFormat = convertSnowflakeFormatToMomentFormat;
+

lib/connection/result/sf_timestamp.js

@@ -4,36 +4,7 @@
 
 const Moment = require('moment-timezone');
 const Util = require('../../util');
-
-/**
- * An array of tag mappings to convert a sql format to a moment.js format. If
- * the 2nd element is empty, special code is needed.
- */
-const CONST_TAGS =
-  [
-    // proper mappings
-    ['YYYY', 'YYYY'],
-    ['YY', 'YY'],
-    ['MM', 'MM'],
-    ['MON', 'MMM'],
-    ['DD', 'DD'],
-    ['DY', 'ddd'],
-    ['HH24', 'HH'],
-    ['HH12', 'hh'],
-    ['HH', 'HH'],
-    ['AM', 'A'],
-    ['PM', 'A'],
-    ['MI', 'mm'],
-    ['SS', 'ss'],
-    ['TZH:TZM', 'Z'],
-    ['TZHTZM', 'ZZ'],
-
-    // special code needed
-    ['TZH', ''],
-    ['TZM', ''],
-    ['FF', '']
-  ];
-
+const datetimeFormatConverter = require('./datetime_format_converter');
 /**
  * Creates a new SfTimestamp instance.
  *
@@ -94,7 +65,7 @@
   // get an upper-case version of the input sql format
   const formatSqlUpper = formatSql.toUpperCase();
 
-  const tags = CONST_TAGS;
+  const tags = datetimeFormatConverter.formatTagsMap();
 
   // iterate over the format string
   const length = formatSql.length;
@@ -160,10 +131,17 @@
       pos += tag.length;
     }
   }
-
   // format the moment and cache the result
-  this._valueAsString = moment.format(formatMoment);
-
+  const timezone = this.timezone.name || this.timezone;
+  if (timezone) {
+    if (typeof timezone === 'number') {
+      this._valueAsString = moment.utcOffset(timezone).format(formatMoment);
+    } else {
+      this._valueAsString = moment.tz(timezone).format(formatMoment);
+    }
+  } else {
+    this._valueAsString = moment.format(formatMoment);
+  }
   return this._valueAsString;
 };