fix: 修改LogisticRegression分类算法，统一为pipelineModel，本地测试预测成功

java_compile/src/main/java/com/example/services/ModelService.java

@@ -265,12 +265,12 @@ public class ModelService {
                  regParam: Float,
                  elasticNetParam: Float
             * */
-                    LogisticRegressionModel lrModel = lrClassification.lrTraining(libsvm,
+                    PipelineModel lrModel = lrClassification.lrTraining(libsvm,
                             Integer.parseInt(modelArguments.get("MaxIter")),
                             Float.parseFloat(modelArguments.get("RegParam")),
                             Float.parseFloat(modelArguments.get("ElasticNetParam")));
                     log.info("\n\nstore Model result\n\n");
-                    HashMap<String,String> lrRst = lrClassification.lrResult(lrModel,modelId);
+                    HashMap<String,String> lrRst = lrClassification.dtResult(lrModel,modelId);
 
                     log.info("\n\nupdate result of model\n\n");
                     rsl = updateResOfModel(modelId, lrRst, lrModel);

java_compile/src/main/scala/com/example/data/FitTestData.scala

@@ -115,22 +115,7 @@ class FitTestData extends SparkConnect{
 
     //modelPath : hdfsServer + "_" + modelId + ".model"
     val modelPath = model.getModelPath
-    if (model.getModelTypeId==1) {
-      val model1=LogisticRegressionModel.load(modelPath)
-      val prections = model1.transform(data)
-
-      prections.show()
-
-      val results = prections.select("predictedLabel")
-      val columns = keyFeature
-      val res = results.collect().map(row => {
-        row.toSeq.zipWithIndex.map(pair => {
-          (columns, pair._1.toString)
-        }).toMap.asJava
-      }).toList.asJava
-
-      res
-    }else if (model.getModelTypeId==5){
+    if (model.getModelTypeId==5){
       val model1 = KMeansModel.load(modelPath)
       val prections = model1.transform(data)
 

java_compile/src/main/scala/com/example/model/classification/LRClassification.scala

@@ -1,10 +1,13 @@
 package com.example.model.classification
 
 import java.util
-
 import com.example.SparkConnect
 import com.example.model.helper.Utils
-import org.apache.spark.ml.classification.{BinaryLogisticRegressionSummary, LogisticRegression, LogisticRegressionModel}
+import org.apache.spark.ml.{Pipeline, PipelineModel}
+import org.apache.spark.ml.classification.{BinaryLogisticRegressionSummary, DecisionTreeClassificationModel, LogisticRegression, LogisticRegressionModel}
+import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator
+import org.apache.spark.ml.feature.{IndexToString, StringIndexer, VectorIndexer}
+import org.apache.spark.sql.{Dataset, Row}
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
@@ -17,38 +20,75 @@ import collection.JavaConverters._
 @Component
 class LRClassification extends SparkConnect{
 
+  var trainingSet: Dataset[Row] = _
+  var testData: Dataset[Row] = _
   @Autowired
   val utils: Utils = null
 
   /*
   * initial params
   * */
+
   /**
-    *
-    * @param libsvmFile
-    * @param maxIter
-    * @param regParam
-    * @param elasticNetParam
-    * @throws java.lang.Exception
-    * @return
-    */
+   *
+   * @param libsvmFile
+   * @param maxIter
+   * @param regParam
+   * @param elasticNetParam
+   * @throws java.lang.Exception
+   * @return
+   */
   @throws(classOf[Exception])
   def lrTraining(libsvmFile: String,
                  maxIter: Int,
                  regParam: Float,
-                 elasticNetParam: Float) : LogisticRegressionModel = {
+                 elasticNetParam: Float): PipelineModel = {
     val data = sparkSession.read.format("libsvm").load(libsvmFile)
     data.printSchema()
     data.show()
+
+
+    // Index labels, adding metadata to the label column.
+    // Fit on whole dataset to include all labels in index.
+    val labelIndexer = new StringIndexer()
+      .setInputCol("label")
+      .setOutputCol("indexedLabel")
+      .fit(data)
+
+    // Automatically identify categorical features, and index them.
+    val featureIndexer = new VectorIndexer()
+      .setInputCol("features")
+      .setOutputCol("indexedFeatures")
+      .setMaxCategories(5)
+      .fit(data)
+
+    data.show()
+
+    // Split the data into training and test sets (30% held out for testing).
+    val Array(trainingSet, testSet) = data.randomSplit(Array(0.7, 1 - 0.7))
+    testData = testSet
+
     val lr = new LogisticRegression()
+      .setLabelCol("indexedLabel")
+      .setFeaturesCol("indexedFeatures")
       .setMaxIter(maxIter)
       .setRegParam(regParam)
       .setElasticNetParam(elasticNetParam)
 
-    // Fit the model
-    val lrModel = lr.fit(data)
-    //println(s"Coefficients: ${lrModel.coefficients} Intercept: ${lrModel.intercept}")
-    lrModel
+    // Convert indexed labels back to original labels.
+    val labelConverter = new IndexToString()
+      .setInputCol("prediction")
+      .setOutputCol("predictedLabel")
+      .setLabels(labelIndexer.labels)
+
+    // Chain indexers and tree in a Pipeline.
+    val pipeline = new Pipeline()
+      .setStages(Array(labelIndexer, featureIndexer, lr, labelConverter))
+
+    // Train model. This also runs the indexers.
+    val model = pipeline.fit(trainingSet)
+
+    model
   }
 
   /**
@@ -58,26 +98,37 @@ class LRClassification extends SparkConnect{
     * @return
     */
   @throws(classOf[Exception])
-  def lrResult(lrModel: LogisticRegressionModel, modelId: Int): util.HashMap[String, String] = {
+  def dtResult(pipelineModel: PipelineModel, modelId: Int): util.HashMap[String, String] = {
     val rslMap = new util.HashMap[String, String]()
-    val trainingSummary = lrModel.summary
+    // Make predictions.
+    // Make predictions.
+    val predictions = pipelineModel.transform(testData)
+
+
+    // Select (prediction, true label) and compute test error.
+    val evaluator = new MulticlassClassificationEvaluator()
+      .setLabelCol("indexedLabel")
+      .setPredictionCol("prediction")
+
+    val weightedRecall = evaluator.setMetricName("weightedRecall").evaluate(predictions);
+    val accuracy = evaluator.setMetricName("accuracy").evaluate(predictions)
+    //println("Test Error = " + (1.0 - accuracy))
+    log.info("\n\nTest Error = " + (1.0 - accuracy));
+    val weightedPrecision = evaluator.setMetricName("weightedPrecision").evaluate(predictions);
+
+    val f1 = evaluator.setMetricName("f1").evaluate(predictions);
 
-    // Obtain the objective per iteration.
-    val objectiveHistory = trainingSummary.objectiveHistory
-    objectiveHistory.foreach(loss => println(loss+"123333"))
 
-    // Obtain the metrics useful to judge performance on test data.
-    // We cast the summary to a BinaryLogisticRegressionSummary since the problem is a
-    // binary classification problem.
-    val binarySummary = trainingSummary.asInstanceOf[BinaryLogisticRegressionSummary]
+    val model = pipelineModel.stages(2).asInstanceOf[LogisticRegressionModel]
+    //println("Learned classification tree model:\n" + treeModel.toDebugString)
+    log.info("\n\nLearned classification Logistic model:\n" + model.toString());
 
-    // Obtain the receiver-operating characteristic as a dataframe and areaUnderROC.
-    val roc = binarySummary.roc
-    roc.show()
 
+    rslMap.put("Accuracy: ", accuracy.toString)
+    rslMap.put("weightedPrecision", weightedPrecision.toString)
+    rslMap.put("weightedRecall", weightedRecall.toString)
+    rslMap.put("f1", f1.toString)
 
-    println(binarySummary.areaUnderROC)
-    rslMap.put("areaUnderROC", binarySummary.areaUnderROC.toString)
 
     return rslMap
   }

java_compile/src/main/scala/com/example/model/helper/Utils.scala

@@ -46,11 +46,6 @@ class Utils extends SparkConnect{
         flag = model.save(modelPath)
         log.info("导出KMeansModel, flag="+flag.toString)
       }
-      case "LogisticRegressionModel" => {
-        val model = anyModel.asInstanceOf[LogisticRegressionModel]
-        flag = model.save(modelPath)
-        log.info("导出LogisticRegressionModel, flag="+flag.toString)
-      }
       case _ =>{
         log.error("没有匹配的model类")
       }