|
|
@@ -64,10 +64,8 @@ class CsvAdapter extends SparkConnect{
|
|
|
val df = sparkSession.read
|
|
|
.option("header", "true")
|
|
|
.option("inferSchema", "true")
|
|
|
- .option("encoding","GBK")
|
|
|
.csv(csvHdfsLoc)
|
|
|
.persist()
|
|
|
-
|
|
|
/*//-> 2. Preprocessing file(String to Int & nulls fill)
|
|
|
val strFeatures = df.schema.filter(_.dataType.equals(StringType)).map(_.name)
|
|
|
|