remove unwanted rows and support multiple columns for vertical data type

alabuel · Sep 7, 2021 · 5b54365 · 5b54365
1 parent 55a8b17
commit 5b54365
Show file tree

Hide file tree

Showing 2 changed files with 84 additions and 37 deletions.
diff --git a/config/data_source_configuration_workbook.go b/config/data_source_configuration_workbook.go
@@ -82,12 +82,8 @@ func dataSourceConfigurationItemRead(ctx context.Context, d *schema.ResourceData
 	orientation := d.Get("orientation").(string)
 	start_column := d.Get("col_start").(string)
 	end_column := d.Get("col_end").(string)
-	var filters []map[string]interface{}
 
-	// make sure csv and excel is not on the same resource
-	if csv_string != "" && excel_file != "" {
-		return diag.FromErr(fmt.Errorf(fmt.Sprintf("%v", "Cannot use csv and excel on the same resource")))
-	}
+	var filters []map[string]interface{}
 
 	// gather all filters
 	if v, ok := d.GetOk("filter"); ok {
@@ -104,8 +100,21 @@ func dataSourceConfigurationItemRead(ctx context.Context, d *schema.ResourceData
 		configuration_item = sheet_name
 	}
 
-	valid_vertical_orientation := []string{"vertical", "Vertical", "VERTICAL", "vert", "Vert", "VERT", "v", "V"}
-	valid_horizontal_orientation := []string{"horizontal", "Horizontal", "HORIZONTAL", "horiz", "Horiz", "HORIZ", "h", "H"}
+	// ###### Start Validations ######
+
+	// make sure csv or excel is used
+	if csv_string == "" && excel_file == "" {
+		return diag.FromErr(fmt.Errorf(fmt.Sprintf("%v", "Must use csv or excel on the resource")))
+	}
+
+	// make sure csv and excel is not on the same resource
+	if csv_string != "" && excel_file != "" {
+		return diag.FromErr(fmt.Errorf(fmt.Sprintf("%v", "Cannot use csv and excel on the same resource")))
+	}
+
+	orientation = strings.ToLower(orientation)
+	valid_vertical_orientation := []string{"vertical", "vert", "v"}
+	valid_horizontal_orientation := []string{"horizontal", "horiz", "h"}
 	if stringInList(orientation, valid_vertical_orientation) {
 		orientation = "vertical"
 	} else if stringInList(orientation, valid_horizontal_orientation) {
@@ -118,14 +127,20 @@ func dataSourceConfigurationItemRead(ctx context.Context, d *schema.ResourceData
 		return diag.FromErr(fmt.Errorf(fmt.Sprintf("%v", "configuration_item is required if type is vertical")))
 	}
 
+	if orientation == "vertical" && csv_string != "" {
+		return diag.FromErr(fmt.Errorf(fmt.Sprintf("%v", "vertical orientation is only valid for excel")))
+	}
+
+	// ###### End Validations ######
+
 	// check if excel is being used
 	if excel_file != "" {
 		csvstring, err := excelToCSV(excel_file, sheet_name, start_column, end_column, configuration_item, col_config_item, orientation)
 		if err != nil {
-			return diag.FromErr(err)
+			return diag.FromErr(fmt.Errorf(fmt.Sprintf("%v", csvstring)))
 		}
 		csv_string = csvstring
-		// 	return diag.FromErr(fmt.Errorf(fmt.Sprintf("%v", "The configuration item \""+configuration_item+"\" has no data")))
+		// return diag.FromErr(fmt.Errorf(fmt.Sprintf("%v", csvstring)))
 	}
 
 	if csv_string != "" {
@@ -208,7 +223,7 @@ func excelToCSV(excel_file string, sheet_name string, start_column string, end_c
 	// Get all rows
 	rows, err := f.GetRows(sheet_name)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf(fmt.Sprintf("%v", rows))
 	}
 
 	// get the number of columns
@@ -247,33 +262,64 @@ func excelToCSV(excel_file string, sheet_name string, start_column string, end_c
 					}
 				}
 			}
-			csv = append(csv, sb.String())
+
+			// only accept rows that are not empty
+			replacer := strings.NewReplacer(",", "", " ", "", "[]", "", "{}", "", "\"", "")
+			if replacer.Replace(sb.String()) != "" {
+				csv = append(csv, sb.String())
+			}
 		}
 	} else {
+		// Get total number of columns
+		maxcol := 0
+		for _, row := range rows {
+			if len(row) > maxcol {
+				maxcol = len(row)
+			}
+		}
 		var sb strings.Builder
 		sb.WriteString("\"configuration_item\",")
+		fieldcount := 0
 		for idx, row := range rows {
-			if idx == len(rows)-1 {
-				sb.WriteString("\"" + row[0] + "\"")
-			} else {
-				sb.WriteString("\"" + row[0] + "\",")
+			if strings.Trim(row[0], " ") != "" {
+				if idx == len(rows)-1 {
+					sb.WriteString("\"" + row[0] + "\"")
+				} else {
+					sb.WriteString("\"" + row[0] + "\",")
+				}
+				fieldcount++
 			}
 		}
 		csv = append(csv, sb.String())
-		sb.Reset()
-		sb.WriteString("\"" + configuration_item + "\",")
-		for idx, row := range rows {
-			if idx == len(rows)-1 {
-				sb.WriteString("\"" + row[1] + "\"")
-			} else {
-				sb.WriteString("\"" + row[1] + "\",")
+		for i := 1; i < maxcol; i++ {
+			sb.Reset()
+			sb.WriteString("\"" + configuration_item + "\",")
+			for idx, row := range rows {
+				if i > len(row)-1 {
+					if idx < len(rows)-1 {
+						sb.WriteString("\"\",")
+					} else {
+						sb.WriteString("\"\"")
+					}
+				} else {
+					if idx < len(rows)-1 {
+						sb.WriteString("\"" + row[i] + "\",")
+					} else {
+						sb.WriteString("\"" + row[i] + "\"")
+					}
+				}
+			}
+			// only accept rows that are not empty
+			replacer := strings.NewReplacer(",", "", " ", "", "[]", "", "{}", "", "\"", "")
+			if replacer.Replace(sb.String()) != "" {
+				csv = append(csv, sb.String())
 			}
 		}
-		csv = append(csv, sb.String())
 	}
 	if len(csv) == 1 {
 		return "", err
 	}
+	// return strings.Join(csv, "\n"), fmt.Errorf(fmt.Sprintf("%v", csv))
 	return strings.Join(csv, "\n"), err
 }
 
@@ -390,7 +436,7 @@ func reMapData(csv []map[string]string, mapping interface{}, filters []map[strin
 				new_value[k] = value[k]
 			} else if strings.HasPrefix(k, "s_") || strings.HasPrefix(k, "string_") {
 				replacer := strings.NewReplacer("s_", "", "string_", "")
-				new_key := strings.Title(replacer.Replace(k))
+				new_key := replacer.Replace(k)
 				if value[k] != "" {
 					new_value[new_key] = value[k]
 				} else {
@@ -430,8 +476,8 @@ func reMapData(csv []map[string]string, mapping interface{}, filters []map[strin
 				} else {
 					new_value[new_key] = []string{}
 				}
-			} else if strings.HasPrefix(k, "m_") || strings.HasPrefix(k, "map_") {
-				replacer := strings.NewReplacer("m_", "", "map_", "")
+			} else if strings.HasPrefix(k, "m_") || strings.HasPrefix(k, "map_") || strings.HasPrefix(k, "h_") || strings.HasPrefix(k, "hash_") {
+				replacer := strings.NewReplacer("m_", "", "map_", "", "h_", "", "hash_", "")
 				new_key := replacer.Replace(k)
 				if value[k] != "" {
 					vlist := strings.Split(value[k], ",")

diff --git a/docs/data-sources/workbook.md b/docs/data-sources/workbook.md
@@ -10,10 +10,10 @@ description: |-
 
 ### Example - Using a CSV
 
-```hcl
+```terraform
 data "config_workbook "csv" {
   csv = <<-EOT
-  configuration_item,attr1,attr2,attr3
+  configuration_item,name,b_create,cidr_block
   vpc,my_vpc,1,"10.0.0.0/16"
   EOT
 }
@@ -30,7 +30,7 @@ data "config_workbook" "csv_file" {
 
 ### Example - Using an Excel file
 
-```hcl
+```terraform
 data "config_workbook" "excel" {
   excel = "filename.xlsx"
   worksheet = "Sheet1"
@@ -44,13 +44,13 @@ data "config_workbook" "excel_vertical" {
   excel = "filename.xlsx"
   worksheet = "Sheet2"
   orientation = "vertical"
-  col_config_item = "vertical_data"
+  configuration_item = "vertical_data"
 }
 ```
 
 ### Example - Using a CSV with a schema
 
-```hcl
+```terraform
 data "config_workbook" "csv_using_yaml" {
   csv = file("filename.csv")
   schema = file("schema.yaml")
@@ -63,7 +63,7 @@ data "config_workbook" "csv_using_json" {
 ```
 
 ### Example - Using an Excel with a schema
-```hcl
+```terraform
 data "config_workbook" "excel_using_yaml" {
   excel = "filename.xlsx"
   worksheet = "Sheet1"
@@ -80,7 +80,7 @@ data "config_workbook" "excel_using_json" {
 ### Schema format - Example 1
 ```yaml
 # you can set the attribute types
-schema_config:
+config_schema:
   vpc:
     attr1:
       name: name
@@ -93,7 +93,7 @@ schema_config:
       type: string
 
 # this format assumes that all attributes are of "string" types
-schema_config:
+config_schema:
   vpc:
     attr1: name
     attr2: create
@@ -105,7 +105,7 @@ schema_config:
 ```json
 // you can set the attribute types
 {
-  "schema_config": {
+  "config_schema": {
     "vpc": {
       "attr1": {
         "name": "name",
@@ -125,7 +125,7 @@ schema_config:
 
 // this format assumes that all attributes are of "string" types
 {
-  "schema_config": {
+  "config_schema": {
     "vpc": {
       "attr1": "name",
       "attr2": "create",
@@ -148,8 +148,9 @@ schema_config:
 3. You can preset the type of the attribute using prefixes.
     - `s_` or `string_`
     - `n_` or `num_` or `number_` or `numeric_`
-    - `b_` or `bool_` or `boolean`
+    - `b_` or `bool_` or `boolean_`
     - `m_` or `map_`
+    - `h_` or `hash_`
     - `l_` or `list_`
     - `t_` or `tag_`
     Attributes without prefixes will be treated as string.  Boolean values are (1,yes,true = True; 0,no,false = False)