|
|
@@ -64,8 +64,10 @@ class CsvAdapter extends SparkConnect{
|
|
|
val df = sparkSession.read
|
|
|
.option("header", "true")
|
|
|
.option("inferSchema", "true")
|
|
|
+ .option("encoding","GBK")
|
|
|
.csv(csvHdfsLoc)
|
|
|
.persist()
|
|
|
+
|
|
|
/*//-> 2. Preprocessing file(String to Int & nulls fill)
|
|
|
val strFeatures = df.schema.filter(_.dataType.equals(StringType)).map(_.name)
|
|
|
|