From a31e78ced46a97ed001104809d9e1a9e1e6f8bfd Mon Sep 17 00:00:00 2001 From: weisj <31143295+weisJ@users.noreply.github.com> Date: Tue, 13 Apr 2021 00:54:42 +0200 Subject: [PATCH] Migrate buildSrc from Groovy to Kotlin --- build.gradle.kts | 13 +- buildSrc/build.gradle.kts | 3 +- .../src/main/groovy/CallableLogger.groovy | 14 - ...nloadPrebuiltBinaryFromGitHubAction.groovy | 360 ------------------ buildSrc/src/main/groovy/JniUtils.groovy | 33 -- buildSrc/src/main/groovy/OneTimeLogger.groovy | 45 --- .../src/main/groovy/UberJniJarPlugin.groovy | 72 ---- ...ebuiltBinariesWhenUnbuildablePlugin.groovy | 174 --------- .../kotlin/DownloadPrebuiltBinariesTask.kt | 210 ++++++++++ buildSrc/src/main/kotlin/JniUtils.kt | 23 ++ buildSrc/src/main/kotlin/Loggers.kt | 37 ++ buildSrc/src/main/kotlin/UberJniJarPlugin.kt | 53 +++ ...sePrebuiltBinariesWhenUnbuildablePlugin.kt | 87 +++++ macos/build.gradle.kts | 4 +- windows/build.gradle.kts | 4 +- 15 files changed, 421 insertions(+), 711 deletions(-) delete mode 100644 buildSrc/src/main/groovy/CallableLogger.groovy delete mode 100644 buildSrc/src/main/groovy/DownloadPrebuiltBinaryFromGitHubAction.groovy delete mode 100644 buildSrc/src/main/groovy/JniUtils.groovy delete mode 100644 buildSrc/src/main/groovy/OneTimeLogger.groovy delete mode 100644 buildSrc/src/main/groovy/UberJniJarPlugin.groovy delete mode 100644 buildSrc/src/main/groovy/UsePrebuiltBinariesWhenUnbuildablePlugin.groovy create mode 100644 buildSrc/src/main/kotlin/DownloadPrebuiltBinariesTask.kt create mode 100644 buildSrc/src/main/kotlin/JniUtils.kt create mode 100644 buildSrc/src/main/kotlin/Loggers.kt create mode 100644 buildSrc/src/main/kotlin/UberJniJarPlugin.kt create mode 100644 buildSrc/src/main/kotlin/UsePrebuiltBinariesWhenUnbuildablePlugin.kt diff --git a/build.gradle.kts b/build.gradle.kts index 723327a4..59a0da12 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -107,13 +107,14 @@ allprojects { val githubAccessToken by props("") plugins.withType { - prebuildBinaries { - prebuildLibrariesFolder = "pre-build-libraries" - missingLibraryIsFailure = false - github { - user = "weisj" - repository = "darklaf" + prebuiltBinaries { + prebuiltLibrariesFolder = "pre-build-libraries" + failIfLibraryIsMissing = false + github( + user = "weisj", + repository = "darklaf", workflow = "libs.yml" + ) { branches = listOf("master", "v$projectVersion", projectVersion) accessToken = githubAccessToken manualDownloadUrl = diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index b7e40e69..0b14fadc 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -1,14 +1,15 @@ apply(from= "../gradle/loadProps.gradle.kts") plugins { + `kotlin-dsl` `java-gradle-plugin` - groovy } val nokeeVersion = extra["nokee.version"] dependencies { implementation(platform("dev.nokee:nokee-gradle-plugins:$nokeeVersion")) + implementation(gradleApi()) } repositories { diff --git a/buildSrc/src/main/groovy/CallableLogger.groovy b/buildSrc/src/main/groovy/CallableLogger.groovy deleted file mode 100644 index f8e2b505..00000000 --- a/buildSrc/src/main/groovy/CallableLogger.groovy +++ /dev/null @@ -1,14 +0,0 @@ -import java.util.concurrent.Callable - -class CallableLogger extends OneTimeLogger implements Callable> { - - CallableLogger(Runnable logger) { - super(logger) - } - - @Override - List call() throws Exception { - super.log() - return Collections.emptyList() - } -} diff --git a/buildSrc/src/main/groovy/DownloadPrebuiltBinaryFromGitHubAction.groovy b/buildSrc/src/main/groovy/DownloadPrebuiltBinaryFromGitHubAction.groovy deleted file mode 100644 index 41035afc..00000000 --- a/buildSrc/src/main/groovy/DownloadPrebuiltBinaryFromGitHubAction.groovy +++ /dev/null @@ -1,360 +0,0 @@ -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import groovy.transform.CompileStatic -import org.gradle.api.DefaultTask -import org.gradle.api.GradleException -import org.gradle.api.Transformer -import org.gradle.api.tasks.OutputFile - -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardCopyOption -import java.util.concurrent.locks.ReadWriteLock -import java.util.concurrent.locks.ReentrantReadWriteLock -import java.util.stream.Stream -import java.util.zip.ZipEntry -import java.util.zip.ZipFile - -@CompileStatic -class DownloadPrebuiltBinaryFromGitHubAction extends DefaultTask { - - private static final ReadWriteLock LOCK = new ReentrantReadWriteLock() - - private static final String VERSION_INFO_FILE_NAME = "github_artifact_versions.json" - private static final String TEMP_PATH = "tmp${File.separator}prebuild" - private static final String PRE_BUILD_PATH = "libs${File.separator}prebuild" - - private final OneTimeLogger tokenWarning = new OneTimeLogger.Static({ - error("""No github access token is specified. Latest artifacts will need to be included manually. - |The access token needs to have the 'read-public' property. Specify using: - | -PgithubAccessToken= - |or by setting - | githubAccessToken= - |inside the gradle.properties file. - |""".stripMargin()) - }, "missingTokenWarning") - private final OneTimeLogger useCachedWarning = new OneTimeLogger({ - log("Could not download artifact or artifact information. Using cached version") - }) - - private Map cacheInfo - - private String manualDownloadUrl = "" - private String user - private String repository - private String workflow - private List branches = [] - private boolean missingLibraryIsFailure - private int timeout - - private String githubAccessToken - private String variant - private Optional prebuiltBinary - - @OutputFile - File getPrebuiltBinaryFile() { - if (user == null) throw new GradleException("Github user isn't specified") - if (repository == null) repository = project.name - if (workflow == null) throw new GradleException("Workflow isn't specified") - - if (prebuiltBinary == null) { - if (githubAccessToken == null || githubAccessToken.isEmpty()) { - tokenWarning.log() - } - prebuiltBinary = getExternalBinary(variant) - } - - return prebuiltBinary.orElseGet { - String errorMessage = """Library for $variant could not be downloaded. - |Download it from $manualDownloadUrl - |""".stripMargin() - - if (missingLibraryIsFailure) { - throw new GradleException(format(errorMessage)) - } else { - new OneTimeLogger.Static({ - error(errorMessage) - }, "${variant}-missing").log() - } - return createDirectory(tempFilePath("dummy/")) - } - } - - void setMissingLibraryIsFailure(boolean missingLibraryIsFailure) { - this.missingLibraryIsFailure = missingLibraryIsFailure - } - - void setGithubAccessToken(String githubAccessToken) { - this.githubAccessToken = githubAccessToken - } - - void setVariant(String variant) { - this.variant = variant - } - - void setUser(String user) { - this.user = user - } - - void setRepository(String repository) { - this.repository = repository - } - - void setWorkflow(String workflow) { - this.workflow = workflow - } - - void setManualDownloadUrl(String manualDownloadUrl) { - this.manualDownloadUrl = manualDownloadUrl - } - - void setBranches(List branches) { - this.branches = branches - } - - void setTimeout(int timeout) { - this.timeout = timeout - } - - private Map getCacheInfo() { - if (cacheInfo == null) { - LOCK.readLock().lock() - try { - File cacheInfoFile = getCacheInfoFile() - JsonSlurper jsonParser = new JsonSlurper() - cacheInfo = jsonParser.parseText(cacheInfoFile.text) as Map - } finally { - LOCK.readLock().unlock() - } - } - return cacheInfo - } - - private File getCacheInfoFile() { - String path = preBuildPath(VERSION_INFO_FILE_NAME) - File cacheInfo = new File(path) - if (!cacheInfo.exists()) { - cacheInfo = createFile(path) - cacheInfo << "{}" - } - return cacheInfo - } - - private void writeToCache(String variantName, String timeStamp, File file) { - LOCK.writeLock().lock() - try { - Map cacheInfo = getCacheInfo() - Map entry = [timeStamp: timeStamp, path: file.absolutePath] - cacheInfo.put(variantName, entry) - getCacheInfoFile().write(JsonOutput.prettyPrint(JsonOutput.toJson(cacheInfo))) - } finally { - LOCK.writeLock().unlock() - } - } - - Optional getExternalBinary(String variant) { - Tuple2, Optional> fetchResult = getBinaryDownloadUrl(variant) - Optional downloadInfo = fetchResult.getFirst() - Optional cachedFile = fetchResult.getSecond() - if (cachedFile.isPresent()) { - log("Reusing previously downloaded binary ${cachedFile.map { it.absolutePath }.orElse(null)}") - return cachedFile - } - Optional downloadedFile = downloadInfo.map { - getBinaryFromUrl(variant, it.url).orElse(null) - } - - if (downloadedFile.isPresent()) { - writeToCache(variant, downloadInfo.get()?.timeStamp, downloadedFile.get()) - } else { - info("No file found for variant $variant") - } - - if (downloadedFile.isPresent()) return downloadedFile - return getCachedFile(variant) - } - - private Optional getBinaryFromUrl(String variant, String url) { - File directory = createDirectory(preBuildPath(variant)) - info("Downloading binary for variant '$variant' from $url") - Optional file = downloadZipFile(url, variant).map { unzip(it, directory).findFirst() }.orElse(Optional.empty()) - info("Finished download for variant '$variant'") - return file - } - - private String preBuildPath(String variant) { - return "${project.buildDir}${File.separator}$PRE_BUILD_PATH${File.separator}$variant" - } - - private Optional downloadZipFile(String url, String variant) { - return fetch(url) { - File file = createFile(zipPath(variant)) - Path response = file.toPath() - Files.copy(it.getInputStream(), response, StandardCopyOption.REPLACE_EXISTING) - return new ZipFile(file) - } - } - - private String zipPath(String name) { - return tempFilePath("${name}.zip") - } - - private String tempFilePath(String name) { - return "$project.buildDir${File.separator}$TEMP_PATH${File.separator}${name}" - } - - private static Stream unzip(ZipFile self, File directory) { - Collection files = self.entries().findAll { !(it as ZipEntry).directory } - return files.stream().map { - ZipEntry e = it as ZipEntry - e.name.with { fileName -> - File outputFile = createFile("${directory.path}$File.separator$fileName") - Files.copy(self.getInputStream(e), outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING) - return outputFile - } - } - } - - private Tuple2, Optional> getBinaryDownloadUrl(String variantName) { - boolean isUptoDate = false - File cachedFile = null - String timeStamp = null - String artifactUrl = getLatestRun(getJson(getWorkflowsUrl())).with { - timeStamp = it.get("created_at") - Optional cachedFilePath = getCachedFilePath(variantName, timeStamp) - isUptoDate = cachedFilePath.isPresent() - if (isUptoDate) { - cachedFile = new File(cachedFilePath.get()) - isUptoDate = cachedFile.exists() - } - return get("artifacts_url") as String - } - info("Latest artifact for variant '$variantName' is from $timeStamp") - if (isUptoDate) { - return new Tuple2<>(Optional.empty(), Optional.of(cachedFile)) - } - DownloadInfo downloadInfo = artifactUrl?.with { url -> - Map[] artifacts = getJson(url).get("artifacts") as Map[] - String artifactDownloadUrl = artifacts?.find { variantName == it.get("name") }?.get("url") as String - return artifactDownloadUrl?.with { - new DownloadInfo(getJson(it)?.get("archive_download_url") as String, timeStamp) - } - } - - return new Tuple2<>(Optional.ofNullable(downloadInfo), Optional.empty()) - } - - private String getWorkflowsUrl() { - return "https://api.github.com/repos/$user/$repository/actions/workflows/$workflow/runs" - } - - private Optional getCachedFilePath(String variantName, String timeStamp) { - Map cacheInfo = getCacheInfo() - boolean isLatest = (cacheInfo[variantName] as Map)?.get("timeStamp") == timeStamp - if (isLatest) { - return Optional.ofNullable((cacheInfo[variantName] as Map)?.get("path") as String) - } else { - return Optional.empty() - } - } - - private Optional getCachedFile(String variant) { - Map cacheInfo = getCacheInfo() - return Optional.ofNullable(cacheInfo[variant] as Map).map { - return new File(String.valueOf(it["path"])).with { - if (it.exists()) useCachedWarning.log() - it.exists() ? it : null - } - } - } - - private Map getLatestRun(Map json) { - Map[] runs = json.get("workflow_runs") as Map[] - return Optional.ofNullable(runs?.find { run -> - boolean completed = "completed" == run.get("status") - boolean success = "success" == run.get("conclusion") - boolean isCorrectBranch = branches.isEmpty() || branches.contains(run.get("head_branch")?.toString()) - return completed && success && isCorrectBranch - }).orElseGet { - log("No suitable workflow run found.") - return Collections.emptyMap() - } - } - - private Map getJson(String url) { - return fetch(url) { - JsonSlurper jsonParser = new JsonSlurper() - Map parsedJson = jsonParser.parseText(it.getInputStream().getText()) as Map - return parsedJson - }.orElse(Collections.emptyMap()) - } - - private Optional fetch(String url, Transformer transformer) { - info("Fetching $url") - if (isOffline()) return Optional.empty() - HttpURLConnection get = new URL(url).openConnection() as HttpURLConnection - get.setRequestMethod("GET") - if (timeout >= 0) { - get.setConnectTimeout(timeout) - } - githubAccessToken?.with { - get.setRequestProperty("Authorization", "token $it") - } - try { - def responseCode = get.getResponseCode() - if (responseCode == HttpURLConnection.HTTP_OK) { - return Optional.ofNullable(transformer.transform(get)) - } else { - log("Could not fetch $url. Response code '$responseCode'.") - } - } catch (IOException ignored) { - error(ignored.getMessage()) - } - return Optional.empty() - } - - private static File createFile(String fileName) { - File file = new File(fileName) - if (file.exists()) file.delete() - file.getParentFile().mkdirs() - file.createNewFile() - return file - } - - private static File createDirectory(String fileName) { - File file = new File(fileName) - file.mkdirs() - return file - } - - private boolean isOffline() { - return project.getGradle().startParameter.isOffline() - } - - private void info(String message) { - project.logger.info(format(message)) - } - - private void log(String message) { - project.logger.warn(format(message)) - } - - private void error(String message) { - project.logger.error(format(message)) - } - - private String format(String message) { - String pad = " " * (project.name.size() + 2) - return "${project.name}: ${message.replace("\n", "\n$pad")}" - } - - private class DownloadInfo { - protected String url - protected String timeStamp - - private DownloadInfo(String url, String timeStamp) { - this.url = url - this.timeStamp = timeStamp - } - } -} diff --git a/buildSrc/src/main/groovy/JniUtils.groovy b/buildSrc/src/main/groovy/JniUtils.groovy deleted file mode 100644 index bd5b0d63..00000000 --- a/buildSrc/src/main/groovy/JniUtils.groovy +++ /dev/null @@ -1,33 +0,0 @@ -import dev.nokee.runtime.nativebase.OperatingSystemFamily -import dev.nokee.runtime.nativebase.TargetMachine -import org.gradle.api.GradleException -import org.gradle.api.Project - -class JniUtils { - static String asVariantName(TargetMachine targetMachine) { - String operatingSystemFamily = 'macos' - if (targetMachine.operatingSystemFamily.windows) { - operatingSystemFamily = 'windows' - } else if (targetMachine.operatingSystemFamily.linux) { - operatingSystemFamily = 'linux' - } - - String architecture = 'x86-64' - if (targetMachine.architecture.is32Bit()) { - architecture = 'x86' - } - - return "$operatingSystemFamily-$architecture" - } - - static String getLibraryFileNameFor(Project project, OperatingSystemFamily osFamily) { - if (osFamily.windows) { - return "${project.name}.dll" - } else if (osFamily.linux) { - return "lib${project.name}.so" - } else if (osFamily.macOS) { - return "lib${project.name}.dylib" - } - throw new GradleException("Unknown operating system family '${osFamily}'.") - } -} diff --git a/buildSrc/src/main/groovy/OneTimeLogger.groovy b/buildSrc/src/main/groovy/OneTimeLogger.groovy deleted file mode 100644 index 8a906e10..00000000 --- a/buildSrc/src/main/groovy/OneTimeLogger.groovy +++ /dev/null @@ -1,45 +0,0 @@ -class OneTimeLogger { - private final Runnable logger - private boolean messageAlreadyLogged = false - - OneTimeLogger(Runnable logger) { - this.logger = logger - } - - protected boolean isLogged() { - return messageAlreadyLogged - } - - protected void setLogged(boolean logged) { - messageAlreadyLogged = logged - } - - protected void log() { - if (!isLogged()) { - logger.run() - setLogged(true) - } - } - - static class Static extends OneTimeLogger { - - private static final Map isLogged = new HashMap<>() - - private final Object identifier - - Static(Runnable logger, Object identifier) { - super(logger) - this.identifier = identifier - } - - @Override - protected void setLogged(boolean logged) { - isLogged.put(identifier, logged) - } - - @Override - protected boolean isLogged() { - return Boolean.TRUE == isLogged.get(identifier) - } - } -} diff --git a/buildSrc/src/main/groovy/UberJniJarPlugin.groovy b/buildSrc/src/main/groovy/UberJniJarPlugin.groovy deleted file mode 100644 index 3e4c9f4e..00000000 --- a/buildSrc/src/main/groovy/UberJniJarPlugin.groovy +++ /dev/null @@ -1,72 +0,0 @@ -import dev.nokee.platform.jni.JniJarBinary -import dev.nokee.platform.jni.JniLibrary -import dev.nokee.platform.jni.JniLibraryExtension -import dev.nokee.runtime.nativebase.TargetMachine -import groovy.transform.CompileStatic -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.Transformer -import org.gradle.api.file.CopySpec -import org.gradle.api.provider.Provider -import org.gradle.jvm.tasks.Jar - -@CompileStatic -class UberJniJarPlugin implements Plugin { - - @Override - void apply(Project project) { - project.tasks.named('jar', Jar) { task -> - configure(task) - } - } - - private static void configure(Jar task) { - def project = task.getProject() - def logger = task.getLogger() - def library = project.extensions.getByType(JniLibraryExtension) - library.binaries.withType(JniJarBinary).configureEach { - if (it.jarTask.isPresent()) it.jarTask.get()?.enabled = false - } - logger.info("${project.name}: Merging binaries into the JVM Jar.") - if (library.targetMachines.get().size() > 1) { - for (TargetMachine targetMachine : library.targetMachines.get()) { - Provider variant = library.variants - .flatMap(targetMachineOf(targetMachine)) - .map(onlyOne() as Transformer>) as Provider - task.into(variant.map { it.resourcePath }) { CopySpec spec -> - spec.from(variant.map { it.nativeRuntimeFiles }) - } - } - } else { - library.variants.configureEach { - task.into(it.resourcePath) { CopySpec spec -> - spec.from(it.nativeRuntimeFiles) - } - } - } - } - - // Filter variants that match the specified target machine. - private static Transformer, JniLibrary> targetMachineOf(TargetMachine targetMachine) { - return new Transformer, JniLibrary>() { - @Override - Iterable transform(JniLibrary variant) { - if (variant.targetMachine == targetMachine) { - return [variant] - } - return [] - } - } - } - - // Ensure only a single variant is present in the collection and return the variant. - private static Transformer> onlyOne() { - return new Transformer>() { - @Override - JniLibrary transform(List variants) { - assert variants.size() == 1 - return variants.first() - } - } - } -} diff --git a/buildSrc/src/main/groovy/UsePrebuiltBinariesWhenUnbuildablePlugin.groovy b/buildSrc/src/main/groovy/UsePrebuiltBinariesWhenUnbuildablePlugin.groovy deleted file mode 100644 index 888ee0f1..00000000 --- a/buildSrc/src/main/groovy/UsePrebuiltBinariesWhenUnbuildablePlugin.groovy +++ /dev/null @@ -1,174 +0,0 @@ -import dev.nokee.platform.jni.JniLibraryExtension -import groovy.transform.CompileStatic -import org.gradle.api.Action -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.plugins.ExtensionAware - -@CompileStatic -class UsePrebuiltBinariesWhenUnbuildablePlugin implements Plugin { - - private PrebuildBinariesExtension prebuildExtension - private GithubArtifactExtension githubArtifactExtension - - void prebuildBinaries(Action action) { - action.execute(prebuildExtension) - } - - PrebuildBinariesExtension getPrebuildBinaries() { - return prebuildExtension - } - - @Override - void apply(Project project) { - JniLibraryExtension library = project.extensions.getByType(JniLibraryExtension) - prebuildExtension = project.extensions.create("prebuildBinaries", PrebuildBinariesExtension, this) - githubArtifactExtension = (prebuildExtension as ExtensionAware).with { - it.extensions.create("github", GithubArtifactExtension) - } - library.variants.configureEach { var -> - if (prebuildExtension.alwaysUsePrebuildArtifact || !var.sharedLibrary.buildable) { - // Try to include the library file... if available - def defaultLibraryName = JniUtils.getLibraryFileNameFor(project, var.targetMachine.operatingSystemFamily) - def variantName = JniUtils.asVariantName(var.targetMachine) - def libraryFile = project.file( - "${prebuildExtension.prebuildLibrariesFolder}/$variantName/$defaultLibraryName" - ) - - if (!libraryFile.exists()) { - // No local binary provided. Try to download it from github actions. - def prebuiltBinariesTask = project.tasks.register("downloadPrebuiltBinary$variantName", DownloadPrebuiltBinaryFromGitHubAction.class) - prebuiltBinariesTask.configure { - it.githubAccessToken = githubArtifactExtension.accessToken - it.variant = variantName - it.user = githubArtifactExtension.user - it.repository = githubArtifactExtension.repository - it.workflow = githubArtifactExtension.workflow - it.manualDownloadUrl = githubArtifactExtension.manualDownloadUrl - it.branches = githubArtifactExtension.branches - it.missingLibraryIsFailure = prebuildExtension.missingLibraryIsFailure - it.timeout = githubArtifactExtension.timeout - } - var.nativeRuntimeFiles.setFrom(prebuiltBinariesTask.map { it.prebuiltBinaryFile }) - var.nativeRuntimeFiles.from(new CallableLogger({ - project.logger.warn("${project.name}: Using pre-build library from github for targetMachine $variantName.") - })) - } else { - //Use provided library. - var.nativeRuntimeFiles.setFrom(libraryFile) - var.nativeRuntimeFiles.from(new CallableLogger({ - def relativePath = project.rootProject.relativePath(libraryFile) - project.logger.warn("${project.name}: Using pre-build library $relativePath for targetMachine $variantName.") - })) - } - } - } - } - - static class PrebuildBinariesExtension { - - private String prebuildLibrariesFolder = "pre-build-libraries" - private boolean alwaysUsePrebuildArtifact = false - private boolean missingLibraryIsFailure = true - private UsePrebuiltBinariesWhenUnbuildablePlugin plugin - - PrebuildBinariesExtension(UsePrebuiltBinariesWhenUnbuildablePlugin plugin) { - this.plugin = plugin - } - - void github(Action action) { - action.execute(plugin.githubArtifactExtension) - } - - void setAlwaysUsePrebuildArtifact(boolean alwaysUsePrebuildArtifact) { - this.alwaysUsePrebuildArtifact = alwaysUsePrebuildArtifact - } - - boolean getAlwaysUsePrebuildArtifact() { - return alwaysUsePrebuildArtifact - } - - String getPrebuildLibrariesFolder() { - return prebuildLibrariesFolder - } - - boolean getMissingLibraryIsFailure() { - return missingLibraryIsFailure - } - - void setPrebuildLibrariesFolder(String prebuildLibrariesFolder) { - this.prebuildLibrariesFolder = prebuildLibrariesFolder - } - - void setMissingLibraryIsFailure(boolean missingLibraryIsFailure) { - this.missingLibraryIsFailure = missingLibraryIsFailure - } - } - - static class GithubArtifactExtension { - private String user - private String repository - private String workflow - private String manualDownloadUrl - private String accessToken - private int timeout = 0 - private List branches = ["master"] - - String getUser() { - return user - } - - String getRepository() { - return repository - } - - String getWorkflow() { - return workflow - } - - String getManualDownloadUrl() { - return manualDownloadUrl - } - - String getAccessToken() { - return accessToken - } - - List getBranches() { - return branches - } - - int getTimeout() { - return timeout - } - - void setUser(String user) { - this.user = user - } - - void setRepository(String repository) { - this.repository = repository - } - - void setWorkflow(String workflow) { - this.workflow = workflow - } - - void setManualDownloadUrl(String manualDownloadUrl) { - this.manualDownloadUrl = manualDownloadUrl - } - - void setAccessToken(String accessToken) { - this.accessToken = accessToken - } - - void setBranches(List branches) { - this.branches = branches - } - - void setTimeout(int timeout) { - this.timeout = timeout - } - } - -} diff --git a/buildSrc/src/main/kotlin/DownloadPrebuiltBinariesTask.kt b/buildSrc/src/main/kotlin/DownloadPrebuiltBinariesTask.kt new file mode 100644 index 00000000..1c7b9b8f --- /dev/null +++ b/buildSrc/src/main/kotlin/DownloadPrebuiltBinariesTask.kt @@ -0,0 +1,210 @@ +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.tasks.OutputFile +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReadWriteLock +import java.util.concurrent.locks.ReentrantReadWriteLock +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import javax.inject.Inject + +private typealias Json = Map + +@Suppress("UNCHECKED_CAST") +open class DownloadPrebuiltBinariesTask @Inject constructor( + private val variantName: String, + private val extension: PrebuiltBinariesExtension +) : DefaultTask() { + + companion object { + private val LOCK: ReadWriteLock = ReentrantReadWriteLock() + private const val VERSION_INFO_FILE_NAME = "github_artifact_versions.json" + private const val TEMP_PATH = "tmp/prebuild" + private const val PRE_BUILD_PATH = "libs/prebuild" + } + + private val githubArtifactSpec = extension.githubArtifactSpec ?: throw GradleException("Github is not configured.") + + private val isOffline + get() = project.gradle.startParameter.isOffline + + private val workflowURL + get() = with(githubArtifactSpec) { + URL("https://api.github.com/repos/$user/$repository/actions/workflows/$workflow/runs") + } + + private val prebuiltDirectoryPath = "${project.buildDir}/$PRE_BUILD_PATH/$variantName" + + private val cacheFile: File by lazy { + val cachePath = "${project.buildDir}/$PRE_BUILD_PATH/$VERSION_INFO_FILE_NAME" + fileOf(cachePath).also { it.writeText("{}") } + } + private val cache: Json by lazy { LOCK.read { cacheFile.readText().toJson() } } + + private val prebuiltBinary: File? by lazy { fetchBinaryFile() } + + @OutputFile + fun getPrebuiltBinaryFile(): File { + return prebuiltBinary ?: run { + val errorMessage = """ + Library for $variantName could not be downloaded. + Download it from ${githubArtifactSpec.manualDownloadUrl} + """.trimIndent() + if (extension.failIfLibraryIsMissing) { + throw GradleException(errorMessage) + } else { + OneTimeAction.createGlobal("$variantName-missing") { + errorLog(errorMessage) + }.execute() + } + directoryOf(tempFilePath("dummy/")) + } + } + + private fun fetchFailed(message: String = ""): T? { + errorLog(message) + return null + } + + private fun fetchBinaryFile(): File? { + val run = workflowURL.getJson().latestRun + ?: return fetchFailed("Could not get latest run") + val timeStamp = run["created_at"] + val cachedPathTimeStamp = cache["timeStamp"] + infoLog("Latest artifact for variant '$variantName' is from $timeStamp") + if (timeStamp == cachedPathTimeStamp) { + val cachedFile = File(cache["path"].toString()) + if (cachedFile.exists()) { + warnLog("Reusing previously downloaded binary ${cachedFile.absolutePath}") + return cachedFile + } + } + val artifactUrl = run["artifacts_url"]?.toString() + ?: return fetchFailed("Could not get artifacts urls") + val artifacts = URL(artifactUrl).getJson()["artifacts"] as List + val downloadUrl = artifacts.find { variantName == it["name"] }?.get("url")?.toString() + ?: return fetchFailed("Could not find matching artifact for $variantName") + val artifactDownloadUrl = URL(downloadUrl).getJson()["archive_download_url"]?.toString() + ?: return fetchFailed("Could not get download url") + val artifact = downloadBinary(artifactDownloadUrl) + if (artifact != null) { + LOCK.write { + val mutableCache = cache.toMutableMap() + mutableCache["timeStamp"] = timeStamp ?: "" + mutableCache["path"] = artifact.absolutePath + cacheFile.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(mutableCache))) + } + } + return artifact + } + + private fun downloadBinary(url: String): File? { + infoLog("Downloading binary for variant '$variantName' from $url") + return URL(url).fetch { + val artifact = fileOf(tempFilePath("$variantName.zip")) + Files.copy(it.inputStream, artifact.toPath(), StandardCopyOption.REPLACE_EXISTING) + infoLog("Finished download for variant '$variantName'") + ZipFile(artifact).unzip(directoryOf(prebuiltDirectoryPath)).firstOrNull() + } + } + + private val Json.latestRun: Json? + get() { + val runs = this["workflow_runs"] as List + val candidates = runs.asSequence().filter { + val completed = "completed" == it["status"] + val success = "success" == it["conclusion"] + completed && success + } + val branches = githubArtifactSpec.branches + if (branches.isEmpty()) return candidates.firstOrNull() + return branches.asSequence().mapNotNull { branch -> + candidates.find { branch == it["head_branch"] } + }.firstOrNull() + } + + private fun URL.getJson(): Json = fetch { connection -> + connection.inputStream.bufferedReader().use { it.readText() }.toJson() + } ?: emptyMap() + + private fun URL.fetch(transform: (HttpURLConnection) -> T?): T? { + if (isOffline) return null + infoLog("Fetching $this") + (openConnection() as HttpURLConnection).run { + requestMethod = "GET" + if (githubArtifactSpec.timeout >= 0) { + connectTimeout = githubArtifactSpec.timeout + } + githubArtifactSpec.accessToken?.also { + setRequestProperty("Authorization", "token $it") + } + return runCatching { + when (responseCode) { + HttpURLConnection.HTTP_OK -> return transform(this) + else -> error("Could not fetch $url. Response code '$responseCode'.") + } + }.getOrElse { + errorLog(it.message ?: "") + null + } + } + } + + private fun tempFilePath(name: String) = "${project.buildDir}/$TEMP_PATH/${name}" + + private fun directoryOf(fileName: String) = File(fileName).also { it.mkdirs() } + + private fun fileOf(fileName: String): File { + val file = File(fileName) + if (!file.exists()) { + file.parentFile.mkdirs() + file.createNewFile() + } + return file + } + + private fun infoLog(message: String) = project.logger.info(message.format()) + private fun warnLog(message: String) = project.logger.warn(message.format()) + private fun errorLog(message: String) = project.logger.error(message.format()) + + private fun String.format(): String { + val pad = " ".repeat(project.name.length + 2) + return "${project.name}: ${replace("\n", "\n$pad")}" + } + + private fun String.toJson(): Json = + JsonSlurper().parseText(this) as Json + + private fun Lock.use(action: () -> T): T { + lock() + try { + return action() + } catch (e: Exception) { + unlock() + throw e + } finally { + unlock() + } + } + + private fun ReadWriteLock.read(action: () -> T): T = readLock().use(action) + private fun ReadWriteLock.write(action: () -> T): T = writeLock().use(action) + + private fun ZipFile.unzip(directory: File): Sequence { + return entries().asSequence() + .map { it as ZipEntry } + .filter { !it.isDirectory } + .map { + val entryFile = fileOf("${directory.path}/${it.name}") + Files.copy(getInputStream(it), entryFile.toPath(), StandardCopyOption.REPLACE_EXISTING) + entryFile + } + } +} diff --git a/buildSrc/src/main/kotlin/JniUtils.kt b/buildSrc/src/main/kotlin/JniUtils.kt new file mode 100644 index 00000000..d7f59fcb --- /dev/null +++ b/buildSrc/src/main/kotlin/JniUtils.kt @@ -0,0 +1,23 @@ +import dev.nokee.runtime.nativebase.OperatingSystemFamily +import dev.nokee.runtime.nativebase.TargetMachine +import org.gradle.api.GradleException +import org.gradle.api.Project + +val TargetMachine.variantName: String + get() { + val osFamily = when { + operatingSystemFamily.isWindows -> "windows" + operatingSystemFamily.isLinux -> "linux" + operatingSystemFamily.isMacOS -> "macos" + else -> GradleException("Unknown operating system family '${operatingSystemFamily}'.") + } + val architecture = if (architecture.is32Bit) "x86" else "x86-64" + return "$osFamily-$architecture" + } + +fun libraryFileNameFor(project : Project, osFamily: OperatingSystemFamily) : String = when { + osFamily.isWindows -> "${project.name}.dll" + osFamily.isLinux -> "lib${project.name}.so" + osFamily.isMacOS -> "lib${project.name}.dylib" + else -> throw GradleException("Unknown operating system family '${osFamily}'.") +} diff --git a/buildSrc/src/main/kotlin/Loggers.kt b/buildSrc/src/main/kotlin/Loggers.kt new file mode 100644 index 00000000..c89ba384 --- /dev/null +++ b/buildSrc/src/main/kotlin/Loggers.kt @@ -0,0 +1,37 @@ +import java.io.File +import java.util.concurrent.Callable + +class CallableAction(action: () -> Unit) : OneTimeAction(action), Callable> { + + override fun call(): List { + this.execute() + return emptyList() + } +} + + +open class OneTimeAction(private val action: () -> Unit) { + internal open var alreadyExecuted = false + + fun execute() { + if (alreadyExecuted) return + alreadyExecuted = true + action() + } + + companion object { + + private val isExecutedMap = mutableMapOf() + + fun createGlobal(name: String, action: () -> Unit): OneTimeAction { + isExecutedMap.putIfAbsent(name, false) + return object : OneTimeAction(action) { + override var alreadyExecuted + get() = isExecutedMap[name] ?: false + set(value) { + isExecutedMap[name] = value + } + } + } + } +} diff --git a/buildSrc/src/main/kotlin/UberJniJarPlugin.kt b/buildSrc/src/main/kotlin/UberJniJarPlugin.kt new file mode 100644 index 00000000..30fafc33 --- /dev/null +++ b/buildSrc/src/main/kotlin/UberJniJarPlugin.kt @@ -0,0 +1,53 @@ +import dev.nokee.platform.base.VariantView +import dev.nokee.platform.jni.JniJarBinary +import dev.nokee.platform.jni.JniLibrary +import dev.nokee.platform.jni.JniLibraryExtension +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.provider.Provider +import org.gradle.jvm.tasks.Jar +import dev.nokee.runtime.nativebase.TargetMachine + +class UberJniJarPlugin : Plugin { + + override fun apply(target: Project) { + target.tasks.named("jar", Jar::class.java) { + configure(this) + } + } + + private fun configure(task: Jar) { + val project = task.project + val logger = task.logger + val library = project.extensions.getByType(JniLibraryExtension::class.java) + library.binaries.withType(JniJarBinary::class.java).configureEach { + jarTask.configure { enabled = false } + } + logger.info("${project.name}: Merging binaries into the JVM Jar.") + when (library.targetMachines.get().size) { + 0 -> logger.info("No native target for project ${project.name}") + 1 -> { + library.variants.configureEach { + task.into(this@configureEach.resourcePath) { + from(this@configureEach.nativeRuntimeFiles) + } + } + } + else -> { + for (targetMachine in library.targetMachines.get()) { + val variant = library.variants.withTarget(targetMachine) + task.into(variant.map { it.resourcePath }) { + from(variant.map { it.nativeRuntimeFiles }) + } + } + } + } + } + + private fun VariantView.withTarget(target: TargetMachine): Provider { + return filter { it.targetMachine == target }.map { + check(it.size == 1) + it.first() + } + } +} diff --git a/buildSrc/src/main/kotlin/UsePrebuiltBinariesWhenUnbuildablePlugin.kt b/buildSrc/src/main/kotlin/UsePrebuiltBinariesWhenUnbuildablePlugin.kt new file mode 100644 index 00000000..475f839b --- /dev/null +++ b/buildSrc/src/main/kotlin/UsePrebuiltBinariesWhenUnbuildablePlugin.kt @@ -0,0 +1,87 @@ +import org.gradle.api.Action +import org.gradle.api.Plugin +import org.gradle.api.Project +import dev.nokee.platform.jni.JniLibraryExtension +import dev.nokee.platform.jni.JniLibrary +import java.io.File + +class UsePrebuiltBinariesWhenUnbuildablePlugin : Plugin { + + lateinit var prebuiltExtension: PrebuiltBinariesExtension + + fun prebuiltBinaries(action: Action) { + action.execute(prebuiltExtension) + } + + override fun apply(target: Project) { + prebuiltExtension = target.extensions.create("prebuiltBinaries", PrebuiltBinariesExtension::class.java) + val library = target.extensions.getByType(JniLibraryExtension::class.java) + library.variants.configureEach { + if (prebuiltExtension.alwaysUsePrebuiltArtifact || !sharedLibrary.isBuildable) { + configure(target, this) + } + } + } + + private fun configure(project: Project, library: JniLibrary) { + with(prebuiltExtension) { + val defaultLibraryName = libraryFileNameFor(project, library.targetMachine.operatingSystemFamily) + val variantName = library.targetMachine.variantName + val libraryFile = project.file("$prebuiltLibrariesFolder/$variantName/$defaultLibraryName") + + if (libraryFile.exists()) { + useLocalLibrary(project, library, libraryFile, variantName) + } else { + // No local binary provided. Try to download it from github actions. + useGithubLibrary(project, library, variantName) + } + } + } + + private fun useGithubLibrary(project: Project, library: JniLibrary, variantName: String) { + val prebuiltBinariesTask = project.tasks.register( + "downloadPrebuiltBinary$variantName", + DownloadPrebuiltBinariesTask::class.java, + variantName, + prebuiltExtension + ) + library.nativeRuntimeFiles.setFrom(prebuiltBinariesTask.map { it.getPrebuiltBinaryFile() }) + library.nativeRuntimeFiles.from(CallableAction { + project.logger.warn( + "${project.name}: Using pre-build library from github for targetMachine $variantName." + ) + }) + } + + private fun useLocalLibrary(project: Project, library: JniLibrary, libraryFile: File, variantName: String) { + library.nativeRuntimeFiles.setFrom(libraryFile) + library.nativeRuntimeFiles.from(CallableAction { + val relativePath = project.rootProject.relativePath(libraryFile) + project.logger.warn( + "${project.name}: Using pre-build library $relativePath for targetMachine $variantName." + ) + }) + } +} + +open class PrebuiltBinariesExtension { + + internal var githubArtifactSpec: GithubArtifactSpec? = null + var prebuiltLibrariesFolder: String = "pre-build-libraries" + var alwaysUsePrebuiltArtifact: Boolean = false + var failIfLibraryIsMissing: Boolean = true + + fun github(user: String, repository: String, workflow: String, action: Action) { + githubArtifactSpec = GithubArtifactSpec(user, repository, workflow).also { action.execute(it) } + } +} + +data class GithubArtifactSpec( + var user: String, + var repository: String?, + var workflow: String, + var manualDownloadUrl: String = "", + var accessToken: String? = null, + var timeout: Int = 0, + var branches: List = listOf("master") +) diff --git a/macos/build.gradle.kts b/macos/build.gradle.kts index 8c221e94..9cdfec4e 100644 --- a/macos/build.gradle.kts +++ b/macos/build.gradle.kts @@ -1,5 +1,3 @@ -import JniUtils.asVariantName - plugins { java id("dev.nokee.jni-library") @@ -30,7 +28,7 @@ library { targetMachines.addAll(machines.macOS.x86_64) variants.configureEach { - resourcePath.set("com/github/weisj/darklaf/platform/${project.name}/${asVariantName(targetMachine)}") + resourcePath.set("com/github/weisj/darklaf/platform/${project.name}/${targetMachine.variantName}") sharedLibrary { compileTasks.configureEach { compilerArgs.addAll("-mmacosx-version-min=$minOs") diff --git a/windows/build.gradle.kts b/windows/build.gradle.kts index 910a5bdc..a61d1989 100644 --- a/windows/build.gradle.kts +++ b/windows/build.gradle.kts @@ -1,5 +1,3 @@ -import JniUtils.asVariantName - plugins { java id("dev.nokee.jni-library") @@ -20,7 +18,7 @@ library { targetMachines.addAll(machines.windows.x86, machines.windows.x86_64) variants.configureEach { - resourcePath.set("com/github/weisj/darklaf/platform/${project.name}/${asVariantName(targetMachine)}") + resourcePath.set("com/github/weisj/darklaf/platform/${project.name}/${targetMachine.variantName}") sharedLibrary { compileTasks.configureEach { compilerArgs.addAll(toolChain.map {