-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for Alpine on LGBM (#145)
* Add support for Alpine on LGBM This commit bumps lightgbm to v0.9.14 and makes it possible to use LGBM on nodes with AMD64 architecture using Alpine. * Change build structure when loading dependencies This commit changes the paths when loading the dependencies on amd64 architecture in order to reflect the changes made on the jar. * Change build structure when loading dependencies This commit changes the path when loading the dependencies on amd64 architecture in order to reflet the change on the jar structure from amd64/alpine to amd64/musl. * Update make-lightgbm submodule * Refactor code and add tests This commit changes the code structure by adding a new enum representing the libc implementation available. In order to detect the libc implementation, an env variable FDZ_OPENML_JAVA_LIBC must be set with either 'glibc' or 'musl', and using glibc if this env variable is not defined. This new enum is also used along side with the cpu architecture enum to represent the infrastructure running the code. * Add github action to test LightGBM on Alpine * Try action change * Try action change * Try action change * Try action change * Try action change * Add toString method to Infrastructure * Add unit test for Infrastructure * Try action change * Try action change * Try action change * Try action change * Try action change * Try action change * Update READEME This commit includes information in the README about the new ENV variable used to load dependencies for musl-based OS. * Add Override annotation to toString method on Infrastructure Co-authored-by: Artur Pedroso <artpdr@users.noreply.github.com> * Fix toString method on Infrastructure * Add ENV var name as a constant * Make FDZ_OPENML_JAVA_LIBC variable private * Refactor Infrastructure#getLgbmNativeLibsFolder This commits changes the if statment on the method Infrastructure#getLgbmNativeLibsFolder into a switch statement. * Fix unknownLibcImplementationThrowsException test * Remove unnecessary variable on getLgbmNativeLibsFolder * Make getLibcImplementation method protected --------- Co-authored-by: Renato Azevedo <renato.azevedo@feedzai.com> Co-authored-by: Artur Pedroso <artur.pedroso@feedzai.com> Co-authored-by: Artur Pedroso <artpdr@users.noreply.github.com>
- Loading branch information
1 parent
fe58343
commit 4ab3e8e
Showing
11 changed files
with
228 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Submodule make-lightgbm
updated
4 files
+8 −3 | README.md | |
+34 −0 | docker/lightgbm-ci-build-env-alpine/Dockerfile | |
+4 −0 | docker/make_docker_image.sh | |
+20 −4 | make.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
53 changes: 53 additions & 0 deletions
53
.../lightgbm-provider/src/main/java/com/feedzai/openml/provider/lightgbm/Infrastructure.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
package com.feedzai.openml.provider.lightgbm; | ||
|
||
import java.io.IOException; | ||
|
||
|
||
/** | ||
* Enum that represents the infrastructure where code is running and consequent lgbm native libs locations. | ||
*/ | ||
public class Infrastructure { | ||
|
||
/** | ||
* The CPU architecture used. | ||
*/ | ||
private final CpuArchitecture cpuArchitecture; | ||
|
||
/** | ||
* The libc implementation available. | ||
*/ | ||
private final LibcImplementation libcImpl; | ||
|
||
public Infrastructure(final CpuArchitecture cpuArchitecture, final LibcImplementation libcImpl) { | ||
this.cpuArchitecture = cpuArchitecture; | ||
this.libcImpl = libcImpl; | ||
} | ||
|
||
@Override | ||
public String toString() { | ||
return "Infrastructure{" + | ||
"cpuArchitecture=" + cpuArchitecture + | ||
", libcImpl=" + libcImpl + | ||
'}'; | ||
} | ||
|
||
/** | ||
* Gets the native libraries folder name according to the cpu architecture and libc implementation. | ||
* | ||
* @return the native libraries folder name. | ||
*/ | ||
public String getLgbmNativeLibsFolder() throws IOException { | ||
|
||
switch (cpuArchitecture) { | ||
case AARCH64: | ||
if (libcImpl == LibcImplementation.MUSL) { | ||
throw new IOException("Trying to use LightGBM on a musl-based OS with unsupported arm64 architecture."); | ||
} | ||
return cpuArchitecture.getLgbmNativeLibsFolder() + "/"; | ||
case AMD64: | ||
return cpuArchitecture.getLgbmNativeLibsFolder() + "/" + libcImpl.getLibcImpl() + "/"; | ||
default: | ||
throw new IllegalStateException("Unexpected value: " + cpuArchitecture); | ||
} | ||
} | ||
} |
27 changes: 27 additions & 0 deletions
27
...htgbm-provider/src/main/java/com/feedzai/openml/provider/lightgbm/LibcImplementation.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
package com.feedzai.openml.provider.lightgbm; | ||
|
||
/** | ||
* Enum that represents the libc implementation available on the machine and consequent lgbm native libs locations. | ||
*/ | ||
public enum LibcImplementation { | ||
MUSL("musl"), | ||
GLIBC("glibc"); | ||
|
||
/** | ||
* This is the name of available libc implementation and indicates the folder where the lightgbm native libraries are. | ||
*/ | ||
private final String libcImpl; | ||
|
||
LibcImplementation(final String libcImpl){ | ||
this.libcImpl = libcImpl; | ||
} | ||
|
||
/** | ||
* Gets the native libraries folder name according to the libc implementation. | ||
* | ||
* @return the native libraries folder name according to the libc implementation. | ||
*/ | ||
public String getLibcImpl() { | ||
return libcImpl; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
45 changes: 45 additions & 0 deletions
45
...htgbm-provider/src/test/java/com/feedzai/openml/provider/lightgbm/TestInfrastructure.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
/* | ||
* The copyright of this file belongs to Feedzai. The file cannot be | ||
* reproduced in whole or in part, stored in a retrieval system, | ||
* transmitted in any form, or by any means electronic, mechanical, | ||
* photocopying, or otherwise, without the prior permission of the owner. | ||
* | ||
* © 2023 Feedzai, Strictly Confidential | ||
*/ | ||
package com.feedzai.openml.provider.lightgbm; | ||
|
||
import org.assertj.core.api.Assertions; | ||
import org.junit.Test; | ||
|
||
import java.io.IOException; | ||
|
||
/** | ||
* Tests the retrieval of native libs folder path for Infrastructure. | ||
* | ||
* @author Renato Azevedo (renato.azevedo@feedzai.com) | ||
*/ | ||
public class TestInfrastructure { | ||
@Test | ||
public void unknownInfrastructureCombination() { | ||
Infrastructure infrastructure = new Infrastructure(CpuArchitecture.AARCH64, LibcImplementation.MUSL); | ||
|
||
Assertions.assertThatThrownBy(infrastructure::getLgbmNativeLibsFolder) | ||
.isInstanceOf(IOException.class); | ||
} | ||
|
||
@Test | ||
public void knowsCorrectInfrastructureCombination() throws IOException { | ||
|
||
Infrastructure infra_arm64_glibc = new Infrastructure(CpuArchitecture.AARCH64, LibcImplementation.GLIBC); | ||
Assertions.assertThat(infra_arm64_glibc.getLgbmNativeLibsFolder()) | ||
.isEqualTo("arm64/"); | ||
|
||
Infrastructure infra_amd64_glibc = new Infrastructure(CpuArchitecture.AMD64, LibcImplementation.GLIBC); | ||
Assertions.assertThat(infra_amd64_glibc.getLgbmNativeLibsFolder()) | ||
.isEqualTo("amd64/glibc/"); | ||
|
||
Infrastructure infra_amd64_musl = new Infrastructure(CpuArchitecture.AMD64, LibcImplementation.MUSL); | ||
Assertions.assertThat(infra_amd64_musl.getLgbmNativeLibsFolder()) | ||
.isEqualTo("amd64/musl/"); | ||
} | ||
} |
43 changes: 43 additions & 0 deletions
43
...m-provider/src/test/java/com/feedzai/openml/provider/lightgbm/TestLibcImplementation.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
/* | ||
* The copyright of this file belongs to Feedzai. The file cannot be | ||
* reproduced in whole or in part, stored in a retrieval system, | ||
* transmitted in any form, or by any means electronic, mechanical, | ||
* photocopying, or otherwise, without the prior permission of the owner. | ||
* | ||
* © 2023 Feedzai, Strictly Confidential | ||
*/ | ||
package com.feedzai.openml.provider.lightgbm; | ||
|
||
import org.assertj.core.api.Assertions; | ||
import org.junit.Test; | ||
|
||
/** | ||
* Tests the retrieval of libc implementation. | ||
* | ||
* @author Renato Azevedo (renato.azevedo@feedzai.com) | ||
*/ | ||
public class TestLibcImplementation { | ||
@Test | ||
public void unknownLibcImplementationThrowsException() { | ||
Assertions.assertThatThrownBy(() -> LightGBMUtils.getLibcImplementation("klibc")) | ||
.isInstanceOf(IllegalArgumentException.class); | ||
} | ||
|
||
@Test | ||
public void defaultLibcImplementationIsGlibc() { | ||
Assertions.assertThat(LightGBMUtils.getLibcImplementation("")) | ||
.isEqualTo(LibcImplementation.GLIBC); | ||
|
||
Assertions.assertThat(LightGBMUtils.getLibcImplementation(null)) | ||
.isEqualTo(LibcImplementation.GLIBC); | ||
} | ||
|
||
@Test | ||
public void canReadKnownLibcImplementation() { | ||
Assertions.assertThat(LightGBMUtils.getLibcImplementation("glibc")) | ||
.isEqualTo(LibcImplementation.GLIBC); | ||
|
||
Assertions.assertThat(LightGBMUtils.getLibcImplementation("musl")) | ||
.isEqualTo(LibcImplementation.MUSL); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters