From 504ef433b120c23cd2e5fd9056067d262e642575 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Mon, 31 May 2021 10:58:00 +0800 Subject: [PATCH 01/18] update: choose GPU jar --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 32fa4f97ef9..603209105f3 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ libraryDependencies ++= Seq( "com.microsoft.cognitiveservices.speech" % "client-sdk" % "1.14.0", "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", - "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.110", + "com.microsoft.ml.lightgbm" % "lightgbmlib" % "gpu-test", "com.github.vowpalwabbit" % "vw-jni" % "8.9.1", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", ).map(d => d excludeAll (excludes: _*)) From dd8305ceae6ba39b8f0563f26274b305d42fa01c Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Wed, 2 Jun 2021 22:23:17 +0800 Subject: [PATCH 02/18] add deviceType param --- .../ml/spark/lightgbm/LightGBMClassifier.scala | 2 +- .../microsoft/ml/spark/lightgbm/LightGBMRanker.scala | 2 +- .../ml/spark/lightgbm/LightGBMRegressor.scala | 2 +- .../ml/spark/lightgbm/params/LightGBMParams.scala | 7 +++++++ .../ml/spark/lightgbm/params/TrainParams.scala | 10 ++++++---- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala index 57931cb6b0d..b68daee6f4b 100644 --- a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala +++ b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala @@ -53,7 +53,7 @@ class LightGBMClassifier(override val uid: String) getIsUnbalance, getVerbosity, categoricalIndexes, actualNumClasses, getBoostFromAverage, getBoostingType, getLambdaL1, getLambdaL2, getIsProvideTrainingMetric, getMetric, getMinGainToSplit, getMaxDeltaStep, getMaxBinByFeature, getMinDataInLeaf, getSlotNames, - getDelegate, getDartParams(), getExecutionParams(), getObjectiveParams()) + getDelegate, getDartParams(), getExecutionParams(), getObjectiveParams(), getDeviceType) } def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMClassificationModel = { diff --git a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala index 8a4286790e0..914e23cc912 100644 --- a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala +++ b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala @@ -60,7 +60,7 @@ class LightGBMRanker(override val uid: String) getVerbosity, categoricalIndexes, getBoostingType, getLambdaL1, getLambdaL2, getMaxPosition, getLabelGain, getIsProvideTrainingMetric, getMetric, getEvalAt, getMinGainToSplit, getMaxDeltaStep, getMaxBinByFeature, getMinDataInLeaf, getSlotNames, getDelegate, getDartParams(), getExecutionParams(), - getObjectiveParams()) + getObjectiveParams(), getDeviceType) } def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMRankerModel = { diff --git a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRegressor.scala b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRegressor.scala index cdbf2f6e085..5a965d031cc 100644 --- a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRegressor.scala +++ b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRegressor.scala @@ -66,7 +66,7 @@ class LightGBMRegressor(override val uid: String) getFeatureFraction, getMaxDepth, getMinSumHessianInLeaf, numTasks, modelStr, getVerbosity, categoricalIndexes, getBoostFromAverage, getBoostingType, getLambdaL1, getLambdaL2, getIsProvideTrainingMetric, getMetric, getMinGainToSplit, getMaxDeltaStep, getMaxBinByFeature, getMinDataInLeaf, getSlotNames, getDelegate, - getDartParams(), getExecutionParams(), getObjectiveParams()) + getDartParams(), getExecutionParams(), getObjectiveParams(), getDeviceType) } def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMRegressionModel = { diff --git a/src/main/scala/com/microsoft/ml/spark/lightgbm/params/LightGBMParams.scala b/src/main/scala/com/microsoft/ml/spark/lightgbm/params/LightGBMParams.scala index d38b291fc1f..8c6ee5a6e4f 100644 --- a/src/main/scala/com/microsoft/ml/spark/lightgbm/params/LightGBMParams.scala +++ b/src/main/scala/com/microsoft/ml/spark/lightgbm/params/LightGBMParams.scala @@ -459,4 +459,11 @@ trait LightGBMParams extends Wrappable with DefaultParamsWritable with HasWeight this.delegate = Option(delegate) this } + + val deviceType = new Param[String](this, "deviceType", + "Device for the tree learning, you can use GPU to achieve the faster learning") + setDefault(deviceType -> "cpu") + + def getDeviceType: String = $(deviceType) + def setDeviceType(value: String): this.type = set(deviceType, value) } diff --git a/src/main/scala/com/microsoft/ml/spark/lightgbm/params/TrainParams.scala b/src/main/scala/com/microsoft/ml/spark/lightgbm/params/TrainParams.scala index 2f74805d638..aae5f320243 100644 --- a/src/main/scala/com/microsoft/ml/spark/lightgbm/params/TrainParams.scala +++ b/src/main/scala/com/microsoft/ml/spark/lightgbm/params/TrainParams.scala @@ -43,6 +43,7 @@ abstract class TrainParams extends Serializable { def dartModeParams: DartModeParams def executionParams: ExecutionParams def objectiveParams: ObjectiveParams + def deviceType: String override def toString: String = { // Since passing `isProvideTrainingMetric` to LightGBM as a config parameter won't work, @@ -58,7 +59,8 @@ abstract class TrainParams extends Serializable { s"max_delta_step=$maxDeltaStep min_data_in_leaf=$minDataInLeaf ${objectiveParams.toString()} " + (if (categoricalFeatures.isEmpty) "" else s"categorical_feature=${categoricalFeatures.mkString(",")} ") + (if (maxBinByFeature.isEmpty) "" else s"max_bin_by_feature=${maxBinByFeature.mkString(",")} ") + - (if (boostingType == "dart") s"${dartModeParams.toString()}" else "") + (if (boostingType == "dart") s"${dartModeParams.toString()}" else "") + + s"device_type=$deviceType" } } @@ -77,7 +79,7 @@ case class ClassifierTrainParams(parallelism: String, topK: Int, numIterations: maxDeltaStep: Double, maxBinByFeature: Array[Int], minDataInLeaf: Int, featureNames: Array[String], delegate: Option[LightGBMDelegate], dartModeParams: DartModeParams, executionParams: ExecutionParams, - objectiveParams: ObjectiveParams) + objectiveParams: ObjectiveParams, deviceType: String) extends TrainParams { override def toString(): String = { val extraStr = @@ -102,7 +104,7 @@ case class RegressorTrainParams(parallelism: String, topK: Int, numIterations: I maxDeltaStep: Double, maxBinByFeature: Array[Int], minDataInLeaf: Int, featureNames: Array[String], delegate: Option[LightGBMDelegate], dartModeParams: DartModeParams, executionParams: ExecutionParams, - objectiveParams: ObjectiveParams) + objectiveParams: ObjectiveParams, deviceType: String) extends TrainParams { override def toString(): String = { s"alpha=$alpha tweedie_variance_power=$tweedieVariancePower boost_from_average=${boostFromAverage.toString} " + @@ -125,7 +127,7 @@ case class RankerTrainParams(parallelism: String, topK: Int, numIterations: Int, maxDeltaStep: Double, maxBinByFeature: Array[Int], minDataInLeaf: Int, featureNames: Array[String], delegate: Option[LightGBMDelegate], dartModeParams: DartModeParams, executionParams: ExecutionParams, - objectiveParams: ObjectiveParams) + objectiveParams: ObjectiveParams, deviceType: String) extends TrainParams { override def toString(): String = { val labelGainStr = From 80daf313413abcb2bcfebff95673ef2eb3b5367e Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Mon, 7 Jun 2021 13:05:53 +0800 Subject: [PATCH 03/18] Feat: Add GPU version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 603209105f3..663bd8dbb77 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ libraryDependencies ++= Seq( "com.microsoft.cognitiveservices.speech" % "client-sdk" % "1.14.0", "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", - "com.microsoft.ml.lightgbm" % "lightgbmlib" % "gpu-test", + "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.111", "com.github.vowpalwabbit" % "vw-jni" % "8.9.1", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", ).map(d => d excludeAll (excludes: _*)) From 45d264cdfad61fea661c1b2d905c5434e3642271 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Mon, 7 Jun 2021 13:44:50 +0800 Subject: [PATCH 04/18] feat: add GPU support --- .../scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala index 914e23cc912..9d68bc9652a 100644 --- a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala +++ b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala @@ -60,7 +60,8 @@ class LightGBMRanker(override val uid: String) getVerbosity, categoricalIndexes, getBoostingType, getLambdaL1, getLambdaL2, getMaxPosition, getLabelGain, getIsProvideTrainingMetric, getMetric, getEvalAt, getMinGainToSplit, getMaxDeltaStep, getMaxBinByFeature, getMinDataInLeaf, getSlotNames, getDelegate, getDartParams(), getExecutionParams(), - getObjectiveParams(), getDeviceType) + getObjectiveParams(), + getDeviceType) } def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMRankerModel = { From 56d143302dc5c15aa4dfdec8db8466eae8cf5458 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Mon, 7 Jun 2021 17:28:56 +0800 Subject: [PATCH 05/18] feat: add GPU support --- build.sbt | 2 +- .../scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 84eb217382b..c884f887769 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ libraryDependencies ++= Seq( "com.microsoft.cognitiveservices.speech" % "client-sdk" % "1.14.0", "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", - "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.111", + "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.112", "com.github.vowpalwabbit" % "vw-jni" % "8.9.1", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", ).map(d => d excludeAll (excludes: _*)) diff --git a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala index 9d68bc9652a..914e23cc912 100644 --- a/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala +++ b/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala @@ -60,8 +60,7 @@ class LightGBMRanker(override val uid: String) getVerbosity, categoricalIndexes, getBoostingType, getLambdaL1, getLambdaL2, getMaxPosition, getLabelGain, getIsProvideTrainingMetric, getMetric, getEvalAt, getMinGainToSplit, getMaxDeltaStep, getMaxBinByFeature, getMinDataInLeaf, getSlotNames, getDelegate, getDartParams(), getExecutionParams(), - getObjectiveParams(), - getDeviceType) + getObjectiveParams(), getDeviceType) } def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMRankerModel = { From 61430aa0f2145ad3884215264d96d1f0da0a28da Mon Sep 17 00:00:00 2001 From: Wenqing Xu Date: Wed, 9 Jun 2021 23:30:13 +0800 Subject: [PATCH 06/18] update: update lgbm jar version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index c884f887769..0d10df561f6 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ libraryDependencies ++= Seq( "com.microsoft.cognitiveservices.speech" % "client-sdk" % "1.14.0", "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", - "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.112", + "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.110", "com.github.vowpalwabbit" % "vw-jni" % "8.9.1", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", ).map(d => d excludeAll (excludes: _*)) From 72dd1d4594aeeee8af668bfee4c9ec820e96b4ba Mon Sep 17 00:00:00 2001 From: Wenqing Xu Date: Wed, 9 Jun 2021 23:51:25 +0800 Subject: [PATCH 07/18] update: update lgbm jar version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 0d10df561f6..c884f887769 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ libraryDependencies ++= Seq( "com.microsoft.cognitiveservices.speech" % "client-sdk" % "1.14.0", "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", - "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.110", + "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.112", "com.github.vowpalwabbit" % "vw-jni" % "8.9.1", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", ).map(d => d excludeAll (excludes: _*)) From 1583fab6aa953325a671ab5d1e267a0843b55a8e Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Thu, 10 Jun 2021 13:39:41 +0800 Subject: [PATCH 08/18] update: update lightgbm jar package version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index c884f887769..4de8501a591 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ libraryDependencies ++= Seq( "com.microsoft.cognitiveservices.speech" % "client-sdk" % "1.14.0", "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", - "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.112", + "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.113", "com.github.vowpalwabbit" % "vw-jni" % "8.9.1", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", ).map(d => d excludeAll (excludes: _*)) From ebb39924fe48ca0893b4d5ac109eb9dbdac04291 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Thu, 10 Jun 2021 14:49:43 +0800 Subject: [PATCH 09/18] update: update lightgbm jar package version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 4de8501a591..c884f887769 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ libraryDependencies ++= Seq( "com.microsoft.cognitiveservices.speech" % "client-sdk" % "1.14.0", "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", - "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.113", + "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.112", "com.github.vowpalwabbit" % "vw-jni" % "8.9.1", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", ).map(d => d excludeAll (excludes: _*)) From 287bb920271cfa4a7c7d33ac6abe7b9f290904c4 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Tue, 29 Jun 2021 11:13:29 +0800 Subject: [PATCH 10/18] update: changed LightGBM version --- build.sbt | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.sbt b/build.sbt index ddae88b996e..d01fd15d6e5 100644 --- a/build.sbt +++ b/build.sbt @@ -34,8 +34,6 @@ val extraDependencies = Seq( "com.jcraft" % "jsch" % "0.1.54", "org.apache.httpcomponents" % "httpclient" % "4.5.6", "org.apache.httpcomponents" % "httpmime" % "4.5.6", - "com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.112", - "com.github.vowpalwabbit" % "vw-jni" % "8.9.1", "com.linkedin.isolation-forest" %% "isolation-forest_3.0.0" % "1.0.1", ).map(d => d excludeAll (excludes: _*)) val dependencies = coreDependencies ++ extraDependencies From be64c219666ddd0b52d9f5426ed5ea42f91a66c5 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Tue, 29 Jun 2021 16:43:26 +0800 Subject: [PATCH 11/18] update: update LightGBM Jar version --- build.sbt | 2 +- .../microsoft/ml/spark/lightgbm/LightGBMUtils.scala | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index d01fd15d6e5..33d81d0445d 100644 --- a/build.sbt +++ b/build.sbt @@ -219,7 +219,7 @@ lazy val lightgbm = (project in file("lightgbm")) .enablePlugins(SbtPlugin) .dependsOn(core % "test->test;compile->compile") .settings((settings ++ Seq( - libraryDependencies += ("com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.110"), + libraryDependencies += ("com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.114"), name := "mmlspark-lightgbm" )): _*) diff --git a/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala b/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala index d5db122f871..7f3a0c3371b 100644 --- a/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala +++ b/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala @@ -47,8 +47,14 @@ object LightGBMUtils { */ def initializeNativeLibrary(): Unit = { val osPrefix = NativeLoader.getOSPrefix - new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbm") - new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbm_swig") + if (System.getProperty("os.version").toLowerCase.contains("5.4.0")) { + new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbmcuda") + new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbmcuda_swig") + } + else { + new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbm") + new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbm_swig") + } } def getFeaturizer(dataset: Dataset[_], labelColumn: String, featuresColumn: String, From ff7d199be88cb39c7cdc721e847ce056b8f0e6b1 Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Tue, 29 Jun 2021 17:14:07 +0800 Subject: [PATCH 12/18] update: add MMLSpark resolver --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 33d81d0445d..6a00a7e7ac5 100644 --- a/build.sbt +++ b/build.sbt @@ -220,6 +220,7 @@ lazy val lightgbm = (project in file("lightgbm")) .dependsOn(core % "test->test;compile->compile") .settings((settings ++ Seq( libraryDependencies += ("com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.114"), + resolvers += "LightGBM" at "https://mmlspark.blob.core.windows.net/maven/", name := "mmlspark-lightgbm" )): _*) From c0d3d6de8e6b6880e0f6df573e922596785deddf Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Wed, 30 Jun 2021 10:59:49 +0800 Subject: [PATCH 13/18] update: remove mmlspark resolver --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 6a00a7e7ac5..4605f6dd9e9 100644 --- a/build.sbt +++ b/build.sbt @@ -220,7 +220,7 @@ lazy val lightgbm = (project in file("lightgbm")) .dependsOn(core % "test->test;compile->compile") .settings((settings ++ Seq( libraryDependencies += ("com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.114"), - resolvers += "LightGBM" at "https://mmlspark.blob.core.windows.net/maven/", + resolvers += speechResolver, name := "mmlspark-lightgbm" )): _*) From db9e92642ddf298d94cf9d4da126f9370bf18c1d Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Fri, 20 Aug 2021 15:46:43 +0800 Subject: [PATCH 14/18] fix: fix build.sbt --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 7c3fd0e8ff2..6103746384f 100644 --- a/build.sbt +++ b/build.sbt @@ -221,7 +221,7 @@ lazy val deepLearning = (project in file("deep-learning")) lazy val lightgbm = (project in file("lightgbm")) .enablePlugins(SbtPlugin) .dependsOn(core % "test->test;compile->compile") - .settings((settings ++ Seq( + .settings(settings ++ Seq( libraryDependencies += ("com.microsoft.ml.lightgbm" % "lightgbmlib" % "3.2.114"), resolvers += speechResolver, name := "mmlspark-lightgbm" From fefa18d7be37a4bb989d274505b3839b6da94f3c Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Fri, 20 Aug 2021 15:52:57 +0800 Subject: [PATCH 15/18] fix: format fix --- .../com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala b/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala index e3c2836f98a..4c8ebf6fc73 100644 --- a/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala +++ b/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMRanker.scala @@ -60,8 +60,8 @@ class LightGBMRanker(override val uid: String) getFeatureFraction, getMaxDepth, getMinSumHessianInLeaf, numTasks, modelStr, getVerbosity, categoricalIndexes, getBoostingType, getLambdaL1, getLambdaL2, getMaxPosition, getLabelGain, getIsProvideTrainingMetric, getMetric, getEvalAt, getMinGainToSplit, getMaxDeltaStep, - getMaxBinByFeature, getMinDataInLeaf, getSlotNames, getDelegate, getDartParams, - getExecutionParams ,getObjectiveParams, getDeviceType) + getMaxBinByFeature, getMinDataInLeaf, getSlotNames, getDelegate, getDartParams, + getExecutionParams, getObjectiveParams, getDeviceType) } def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMRankerModel = { From 35c8d9c5013ab3277d3b07cb48a0bb44c3ed469c Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Fri, 20 Aug 2021 16:00:28 +0800 Subject: [PATCH 16/18] fix: format fix --- .../com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala b/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala index fc4944c038b..79b60b4b732 100644 --- a/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala +++ b/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMClassifier.scala @@ -54,7 +54,7 @@ class LightGBMClassifier(override val uid: String) getIsUnbalance, getVerbosity, categoricalIndexes, actualNumClasses, getBoostFromAverage, getBoostingType, getLambdaL1, getLambdaL2, getIsProvideTrainingMetric, getMetric, getMinGainToSplit, getMaxDeltaStep, getMaxBinByFeature, getMinDataInLeaf, getSlotNames, - getDelegate, getDartParams(), getExecutionParams(), getObjectiveParams(), getDeviceType) + getDelegate, getDartParams, getExecutionParams, getObjectiveParams, getDeviceType) } def getModel(trainParams: TrainParams, lightGBMBooster: LightGBMBooster): LightGBMClassificationModel = { From 542fa65a844ac512efdc17bcb4503137112a206e Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Thu, 14 Oct 2021 16:57:34 +0800 Subject: [PATCH 17/18] fix: use synapse default ubuntu version --- .../scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala b/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala index eebafcf9f03..761b3138ade 100644 --- a/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala +++ b/lightgbm/src/main/scala/com/microsoft/ml/spark/lightgbm/LightGBMUtils.scala @@ -37,7 +37,7 @@ object LightGBMUtils { */ def initializeNativeLibrary(): Unit = { val osPrefix = NativeLoader.getOSPrefix - if (System.getProperty("os.version").toLowerCase.contains("5.4.0")) { + if (System.getProperty("os.version").toLowerCase.equals("4.15.0-1124-azure")) { new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbmcuda") new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbmcuda_swig") } From 202537a32ad728e8e33aa01af36c7f22fd7783ae Mon Sep 17 00:00:00 2001 From: xuwq1993 Date: Mon, 29 Nov 2021 15:29:39 +0800 Subject: [PATCH 18/18] feat: add GPU support --- .../com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala index b2cd74cbfe3..b9a226c42b1 100644 --- a/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala +++ b/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMUtils.scala @@ -36,7 +36,7 @@ object LightGBMUtils { */ def initializeNativeLibrary(): Unit = { val osPrefix = NativeLoader.getOSPrefix - if (System.getProperty("os.version").toLowerCase.equals("4.15.0-1124-azure")) { + if (System.getProperty("os.version").toLowerCase.equals("4.15.0-1126-azure")) { new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbmcuda") new NativeLoader("/com/microsoft/ml/lightgbm").loadLibraryByName(osPrefix + "_lightgbmcuda_swig") }