updates to spark

aashishdugar · Dec 10, 2018 · 1edf797 · 1edf797
1 parent 32c8633
commit 1edf797
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 5 deletions.
diff --git a/screenshots/taxi/linear_reg1.png b/screenshots/taxi/linear_reg1.png
diff --git a/screenshots/taxi/linear_reg2.png b/screenshots/taxi/linear_reg2.png
diff --git a/source_code/nyc-spark/src/main/scala/PredGreen.scala b/source_code/nyc-spark/src/main/scala/PredGreen.scala
@@ -47,20 +47,26 @@ object PredGreen {
 
     val df = spark.read.schema(FeatureRow)
       .csv(inputPath)
+
+    val featurized = df
       .withColumn("pu_month", month($"pickupTime"))
       .withColumn("pu_dayofyear", dayofyear($"pickupTime"))
       .withColumn("pu_dayofweek", dowUDF($"pickupTime"))
       .withColumn("pu_day", dayofmonth($"pickupTime"))
       .withColumn("pu_hour", hour($"pickupTime"))
       .withColumn("pu_min", minute($"pickupTime"))
 
-    val pred = rfModel.transform(df)
+    val pred = rfModel.transform(featurized)
       .drop("pu_month", "pu_dayofyear", "pu_dayofweek", "pu_day", "pu_hour", "pu_min")
 
-    pred.write
-      .option("header", value = true)
-      .option("timestampFormat", value = "yyyy-MM-dd'T'HH:mm:ss")
-      .csv(outputPath)
+    pred.show()
+
+//    pred
+//        .select("passengers")
+//      .write
+//      .option("header", value = true)
+//      .option("timestampFormat", value = "yyyy-MM-dd'T'HH:mm:ss")
+//      .csv(outputPath)
 
   }