Dynamically infer varchar column name

apps/csv2sql/lib/csv2sql/database/database.ex

@@ -100,13 +100,14 @@ defmodule Csv2sql.Database do
     ProgressTracker.update_row_count(path, length(data_chunk))
   end
 
+  @spec encode_binary(binary()) :: String.t()
   def encode_binary(str) do
     if Helpers.get_config(:remove_illegal_characters) do
       # TODO: Check if utf8mb4 is supported https://github.com/tallakt/codepagex/issues/27
       {:ok, str, replaced} =
         Codepagex.to_string(str, :iso_8859_1, Codepagex.replace_nonexistent(""), 0)
 
-      # TODO: fix this can slow down things
+      # TODO: fix this it can slow down things
       if replaced > 0,
         do: Logger.warn("[#{Process.get(:file)}] Replaced #{replaced} characters in binary data")
 
@@ -116,6 +117,13 @@ defmodule Csv2sql.Database do
     end
   end
 
+  @spec string_column_type(non_neg_integer()) :: :text | {:varchar, non_neg_integer()}
+  def string_column_type(max_data_length) do
+    if max_data_length > varchar_limit(),
+      do: :text,
+      else: {:varchar, max_data_length}
+  end
+
   # Callbacks to implement
   @callback type_mapping(type_map()) :: String.t()
 

apps/csv2sql/lib/csv2sql/database/mysql.ex

@@ -17,8 +17,7 @@ defmodule Csv2sql.Database.MySql do
       type_map[:is_boolean] -> "BIT"
       type_map[:is_integer] -> "INT"
       type_map[:is_float] -> "DOUBLE"
-      type_map[:is_text] -> "LONGTEXT"
-      true -> "VARCHAR(#{varchar_limit()})"
+      true -> type_map[:max_data_length] |> string_column_type() |> get_string_column_type()
     end
   end
 
@@ -54,6 +53,7 @@ defmodule Csv2sql.Database.MySql do
   @spec get_ordering_column_type :: String.t()
   def get_ordering_column_type(), do: "INT UNSIGNED"
 
+  # Private helpers
   defp to_date_string(%DateTime{} = datetime),
     do: datetime |> DateTime.to_date() |> to_datetime_string()
 
@@ -63,4 +63,8 @@ defmodule Csv2sql.Database.MySql do
     do: datetime |> DateTime.to_string() |> String.trim_trailing("Z")
 
   defp to_datetime_string(val), do: val
+
+  defp get_string_column_type(:text), do: "LONGTEXT"
+  defp get_string_column_type({:varchar, 0}), do: "VARCHAR(#{varchar_limit()})"
+  defp get_string_column_type({:varchar, size}), do: "VARCHAR(#{size})"
 end

apps/csv2sql/lib/csv2sql/database/postgres.ex

@@ -17,8 +17,7 @@ defmodule Csv2sql.Database.Postgres do
       type_map[:is_boolean] -> "BOOLEAN"
       type_map[:is_integer] -> "INT"
       type_map[:is_float] -> "NUMERIC(1000, 100)"
-      type_map[:is_text] -> "TEXT"
-      true -> "VARCHAR(#{varchar_limit()})"
+      true -> type_map[:max_data_length] |> string_column_type() |> get_string_column_type()
     end
   end
 
@@ -71,4 +70,9 @@ defmodule Csv2sql.Database.Postgres do
   @impl Csv2sql.Database
   @spec get_ordering_column_type :: String.t()
   def get_ordering_column_type(), do: "INT"
+
+  # Private helpers
+  defp get_string_column_type(:text), do: "TEXT"
+  defp get_string_column_type({:varchar, 0}), do: "VARCHAR(#{varchar_limit()})"
+  defp get_string_column_type({:varchar, size}), do: "VARCHAR(#{size})"
 end