Unpack Skiko in native distributions

Resolves #546

gradle-plugins/compose/src/main/kotlin/org/jetbrains/compose/desktop/application/internal/files/MacJarSignFileCopyingProcessor.kt

@@ -9,7 +9,9 @@ import org.gradle.process.ExecOperations
 import org.jetbrains.compose.desktop.application.internal.MacUtils
 import org.jetbrains.compose.desktop.application.internal.isJarFile
 import org.jetbrains.compose.desktop.application.internal.validation.ValidatedMacOSSigningSettings
-import java.io.*
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.PrintStream
 import java.util.regex.Pattern
 import java.util.zip.ZipEntry
 import java.util.zip.ZipInputStream
@@ -51,33 +53,34 @@ internal class MacJarSignFileCopyingProcessor(
     }
 
     override fun copy(source: File, target: File) {
-        if (!source.isJarFile) {
+        if (source.isJarFile) {
+            signNativeLibsInJar(source, target)
+        } else {
             SimpleFileCopyingProcessor.copy(source, target)
-            return
+            if (source.name.isDylibPath) {
+                signDylib(target)
+            }
+        }
     }
 
+    private fun signNativeLibsInJar(source: File, target: File) {
         if (target.exists()) target.delete()
 
-        ZipInputStream(FileInputStream(source).buffered()).use { zin ->
+        transformJar(source, target) { zin, zout, entry ->
-            ZipOutputStream(FileOutputStream(target).buffered()).use { zout ->
+            if (entry.name.isDylibPath) {
-                copyAndSignNativeLibs(zin, zout)
+                signDylibEntry(zin, zout, entry)
+            } else {
+                zout.withNewEntry(ZipEntry(entry)) {
+                    zin.copyTo(zout)
+                }
             }
         }
     }
 
-    private fun copyAndSignNativeLibs(zin: ZipInputStream, zout: ZipOutputStream) {
+    private fun signDylibEntry(zin: ZipInputStream, zout: ZipOutputStream, sourceEntry: ZipEntry) {
-        for (sourceEntry in generateSequence { zin.nextEntry }) {
-            if (!sourceEntry.name.endsWith(".dylib")) {
-                zout.putNextEntry(ZipEntry(sourceEntry))
-                zin.copyTo(zout)
-            } else {
-
         val unpackedDylibFile = tempDir.resolve(sourceEntry.name.substringAfterLast("/"))
         try {
-                    unpackedDylibFile.outputStream().buffered().use {
+            zin.copyTo(unpackedDylibFile)
-                        zin.copyTo(it)
-                    }
-
             signDylib(unpackedDylibFile)
             val targetEntry = ZipEntry(sourceEntry.name).apply {
                 comment = sourceEntry.comment
@@ -85,18 +88,13 @@ internal class MacJarSignFileCopyingProcessor(
                 method = sourceEntry.method
                 size = unpackedDylibFile.length()
             }
-                    zout.putNextEntry(targetEntry)
+            zout.withNewEntry(ZipEntry(targetEntry)) {
-
+                unpackedDylibFile.copyTo(zout)
-                    unpackedDylibFile.inputStream().buffered().use {
-                        it.copyTo(zout)
             }
         } finally {
             unpackedDylibFile.delete()
         }
     }
-            zout.closeEntry()
-        }
-    }
 
     private fun signDylib(dylibFile: File) {
         val args = arrayListOf(
@@ -121,3 +119,6 @@ internal class MacJarSignFileCopyingProcessor(
         }.assertNormalExitValue()
     }
 }
+
+private val String.isDylibPath
+    get() = endsWith(".dylib")

gradle-plugins/compose/src/main/kotlin/org/jetbrains/compose/desktop/application/internal/files/fileUtils.kt

@@ -5,9 +5,12 @@
 
 package org.jetbrains.compose.desktop.application.internal.files
 
-import java.io.File
+import java.io.*
 import java.security.DigestInputStream
 import java.security.MessageDigest
+import java.util.zip.ZipEntry
+import java.util.zip.ZipInputStream
+import java.util.zip.ZipOutputStream
 
 internal fun fileHash(file: File): String {
     val md5 = MessageDigest.getInstance("MD5")
@@ -23,3 +26,35 @@ internal fun fileHash(file: File): String {
         }
     }
 }
+
+internal inline fun transformJar(
+    sourceJar: File,
+    targetJar: File,
+    fn: (zin: ZipInputStream, zout: ZipOutputStream, entry: ZipEntry) -> Unit
+) {
+    ZipInputStream(FileInputStream(sourceJar).buffered()).use { zin ->
+        ZipOutputStream(FileOutputStream(targetJar).buffered()).use { zout ->
+            for (sourceEntry in generateSequence { zin.nextEntry }) {
+                fn(zin, zout, sourceEntry)
+            }
+        }
+    }
+}
+
+internal inline fun ZipOutputStream.withNewEntry(zipEntry: ZipEntry, fn: () -> Unit) {
+    putNextEntry(zipEntry)
+    fn()
+    closeEntry()
+}
+
+internal fun InputStream.copyTo(file: File) {
+    file.outputStream().buffered().use { os ->
+        copyTo(os)
+    }
+}
+
+internal fun File.copyTo(os: OutputStream) {
+    inputStream().buffered().use { bis ->
+        bis.copyTo(os)
+    }
+}

gradle-plugins/compose/src/main/kotlin/org/jetbrains/compose/desktop/application/tasks/AbstractJPackageTask.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.compose.desktop.application.tasks
 
 import org.gradle.api.file.*
+import org.gradle.api.internal.file.FileOperations
 import org.gradle.api.provider.ListProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.provider.Provider
@@ -17,19 +18,19 @@ import org.gradle.work.InputChanges
 import org.jetbrains.compose.desktop.application.dsl.MacOSSigningSettings
 import org.jetbrains.compose.desktop.application.dsl.TargetFormat
 import org.jetbrains.compose.desktop.application.internal.*
+import org.jetbrains.compose.desktop.application.internal.files.*
 import org.jetbrains.compose.desktop.application.internal.files.MacJarSignFileCopyingProcessor
-import org.jetbrains.compose.desktop.application.internal.files.SimpleFileCopyingProcessor
 import org.jetbrains.compose.desktop.application.internal.files.fileHash
+import org.jetbrains.compose.desktop.application.internal.files.transformJar
 import org.jetbrains.compose.desktop.application.internal.validation.ValidatedMacOSSigningSettings
 import org.jetbrains.compose.desktop.application.internal.validation.validate
-import java.io.File
+import java.io.*
-import java.io.Serializable
 import java.util.*
+import java.util.zip.ZipEntry
 import javax.inject.Inject
 import kotlin.collections.HashMap
 import kotlin.collections.HashSet
-import java.io.ObjectInputStream
+import kotlin.collections.ArrayList
-import java.io.ObjectOutputStream
 
 abstract class AbstractJPackageTask @Inject constructor(
     @get:Input
@@ -181,6 +182,9 @@ abstract class AbstractJPackageTask @Inject constructor(
     @get:LocalState
     protected val signDir: Provider<Directory> = project.layout.buildDirectory.dir("compose/tmp/sign")
 
+    @get:LocalState
+    protected val skikoDir: Provider<Directory> = project.layout.buildDirectory.dir("compose/tmp/skiko")
+
     @get:Internal
     private val libsDir: Provider<Directory> = workingDir.map {
         it.dir("libs")
@@ -197,10 +201,10 @@ abstract class AbstractJPackageTask @Inject constructor(
     override fun makeArgs(tmpDir: File): MutableList<String> = super.makeArgs(tmpDir).apply {
         if (targetFormat == TargetFormat.AppImage || appImage.orNull == null) {
             // Args, that can only be used, when creating an app image or an installer w/o --app-image parameter
-            cliArg("--input", tmpDir)
+            cliArg("--input", libsDir)
             cliArg("--runtime-image", runtimeImage)
 
-            val mappedJar = libsMapping[launcherMainJar.ioFile]
+            val mappedJar = libsMapping[launcherMainJar.ioFile]?.singleOrNull()
                 ?: error("Main jar was not processed correctly: ${launcherMainJar.ioFile}")
             cliArg("--main-jar", mappedJar)
             cliArg("--main-class", launcherMainClass)
@@ -217,6 +221,7 @@ abstract class AbstractJPackageTask @Inject constructor(
             launcherJvmArgs.orNull?.forEach {
                 cliArg("--java-options", "'$it'")
             }
+            cliArg("--java-options", "'-Dskiko.library.path=${'$'}APPDIR'")
             if (currentOS == OS.MacOS) {
                 macDockName.orNull?.let { dockName ->
                     cliArg("--java-options", "'-Xdock:name=$dockName'")
@@ -298,7 +303,9 @@ abstract class AbstractJPackageTask @Inject constructor(
 
             try {
                 for (change in allChanges) {
-                    libsMapping.remove(change.file)?.let { fileOperations.delete(it) }
+                    libsMapping.remove(change.file)?.let { files ->
+                        files.forEach { fileOperations.delete(it) }
+                    }
                     if (change.changeType != ChangeType.REMOVED) {
                         outdatedLibs.add(change.file)
                     }
@@ -315,7 +322,7 @@ abstract class AbstractJPackageTask @Inject constructor(
     }
 
     override fun prepareWorkingDir(inputChanges: InputChanges) {
-        val libsDirFile = libsDir.ioFile
+        val libsDir = libsDir.ioFile
         val fileProcessor =
             withValidatedMacOSSigning { signing ->
                 val tmpDirForSign = signDir.ioFile
@@ -328,18 +335,25 @@ abstract class AbstractJPackageTask @Inject constructor(
                     signing = signing
                 )
             } ?: SimpleFileCopyingProcessor
+        fun copyFileToLibsDir(sourceFile: File): File {
+            val targetFileName =
+                if (sourceFile.isJarFile) "${sourceFile.nameWithoutExtension}-${fileHash(sourceFile)}.jar"
+                else sourceFile.name
+            val targetFile = libsDir.resolve(targetFileName)
+            fileProcessor.copy(sourceFile, targetFile)
+            return targetFile
+        }
 
         val outdatedLibs = invalidateMappedLibs(inputChanges)
         for (sourceFile in outdatedLibs) {
             assert(sourceFile.exists()) { "Lib file does not exist: $sourceFile" }
 
-            val targetFileName =
+            libsMapping[sourceFile] = if (isSkikoForCurrentOS(sourceFile)) {
-                if (sourceFile.isJarFile)
+                val unpackedFiles = unpackSkikoForCurrentOS(sourceFile, skikoDir.ioFile, fileOperations)
-                    "${sourceFile.nameWithoutExtension}-${fileHash(sourceFile)}.jar"
+                unpackedFiles.map { copyFileToLibsDir(it) }
-                else sourceFile.name
+            } else {
-            val targetFile = libsDirFile.resolve(targetFileName)
+                listOf(copyFileToLibsDir(sourceFile))
-            fileProcessor.copy(sourceFile, targetFile)
+            }
-            libsMapping[sourceFile] = targetFile
         }
     }
 
@@ -422,23 +436,25 @@ abstract class AbstractJPackageTask @Inject constructor(
 // Serializable is only needed to avoid breaking configuration cache:
 // https://docs.gradle.org/current/userguide/configuration_cache.html#config_cache:requirements
 private class FilesMapping : Serializable {
-    private var mapping = HashMap<File, File>()
+    private var mapping = HashMap<File, List<File>>()
 
-    operator fun get(key: File): File? =
+    operator fun get(key: File): List<File>? =
         mapping[key]
 
-    operator fun set(key: File, value: File) {
+    operator fun set(key: File, value: List<File>) {
         mapping[key] = value
     }
 
-    fun remove(key: File): File? =
+    fun remove(key: File): List<File>? =
         mapping.remove(key)
 
     fun loadFrom(mappingFile: File) {
         mappingFile.readLines().forEach { line ->
             if (line.isNotBlank()) {
-                val (k, v) = line.split(File.pathSeparatorChar)
+                val paths = line.splitToSequence(File.pathSeparatorChar)
-                mapping[File(k)] = File(v)
+                val lib = File(paths.first())
+                val mappedFiles = paths.drop(1).mapTo(ArrayList()) { File(it) }
+                mapping[lib] = mappedFiles
             }
         }
     }
@@ -448,10 +464,9 @@ private class FilesMapping : Serializable {
         mappingFile.bufferedWriter().use { writer ->
             mapping.entries
                 .sortedBy { (k, _) -> k.absolutePath }
-                .forEach { (k, v) ->
+                .forEach { (k, values) ->
-                    writer.append(k.absolutePath)
+                    (sequenceOf(k) + values.asSequence())
-                    writer.append(File.pathSeparatorChar)
+                        .joinTo(writer, separator = File.pathSeparator, transform = { it.absolutePath })
-                    writer.appendln(v.absolutePath)
                 }
         }
     }
@@ -461,6 +476,39 @@ private class FilesMapping : Serializable {
     }
 
     private fun readObject(stream: ObjectInputStream) {
-        mapping = stream.readObject() as HashMap<File, File>
+        mapping = stream.readObject() as HashMap<File, List<File>>
+    }
+}
+
+private fun isSkikoForCurrentOS(lib: File): Boolean =
+    lib.name.startsWith("skiko-jvm-runtime-${currentOS.id}-${currentArch.id}")
+            && lib.name.endsWith(".jar")
+
+private fun unpackSkikoForCurrentOS(sourceJar: File, skikoDir: File, fileOperations: FileOperations): List<File> {
+    val entriesToUnpack = when (currentOS) {
+        OS.MacOS -> setOf("libskiko-macos-${currentArch.id}.dylib")
+        OS.Linux -> setOf("skiko-windows-${currentArch.id}.dll", "icudtl.dat")
+        OS.Windows -> setOf("libskiko-linux-${currentArch.id}.so")
+    }
+
+    // output files: unpacked libs, corresponding .sha256 files, and target jar
+    val outputFiles = ArrayList<File>(entriesToUnpack.size * 2 + 1)
+    val targetJar = skikoDir.resolve(sourceJar.name)
+    outputFiles.add(targetJar)
+
+    fileOperations.delete(skikoDir)
+    fileOperations.mkdir(skikoDir)
+    transformJar(sourceJar, targetJar) { zin, zout, entry ->
+        // check both entry or entry.sha256
+        if (entry.name.removeSuffix(".sha256") in entriesToUnpack) {
+            val unpackedFile = skikoDir.resolve(entry.name.substringAfterLast("/"))
+            zin.copyTo(unpackedFile)
+            outputFiles.add(unpackedFile)
+        } else {
+            zout.withNewEntry(ZipEntry(entry)) {
+                zin.copyTo(zout)
+            }
+        }
     }
+    return outputFiles
 }

gradle-plugins/compose/src/test/kotlin/org/jetbrains/compose/gradle/DesktopApplicationTest.kt

@@ -8,13 +8,14 @@ package org.jetbrains.compose.gradle
 import org.gradle.internal.impldep.org.testng.Assert
 import org.gradle.testkit.runner.TaskOutcome
 import org.jetbrains.compose.desktop.application.internal.OS
+import org.jetbrains.compose.desktop.application.internal.currentArch
 import org.jetbrains.compose.desktop.application.internal.currentOS
 import org.jetbrains.compose.desktop.application.internal.currentTarget
 import org.jetbrains.compose.test.*
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Assumptions
 import org.junit.jupiter.api.Test
-import java.nio.charset.Charset
+import java.io.File
 import java.util.jar.JarFile
 
 class DesktopApplicationTest : GradlePluginTestBase() {
@@ -190,4 +191,28 @@ class DesktopApplicationTest : GradlePluginTestBase() {
             }
         }
     }
+
+    @Test
+    fun testUnpackSkiko() {
+        with(testProject(TestProjects.unpackSkiko)) {
+            gradle(":runDistributable").build().checks { check ->
+                check.taskOutcome(":runDistributable", TaskOutcome.SUCCESS)
+
+                val libraryPathPattern = "Read skiko library path: '(.*)'".toRegex()
+                val m = libraryPathPattern.find(check.log)
+                val skikoDir = m?.groupValues?.get(1)?.let(::File)
+                if (skikoDir == null || !skikoDir.exists()) {
+                    error("Invalid skiko path: $skikoDir")
+                }
+                val filesToFind = when (currentOS) {
+                    OS.Linux -> listOf("libskiko-linux-${currentArch.id}.so")
+                    OS.Windows -> listOf("skiko-windows-${currentArch.id}.dll", "icudtl.dat")
+                    OS.MacOS -> listOf("libskiko-macos-${currentArch.id}.dylib")
+                }
+                for (fileName in filesToFind) {
+                    skikoDir.resolve(fileName).checkExists()
+                }
+            }
+        }
+    }
 }

gradle-plugins/compose/src/test/kotlin/org/jetbrains/compose/test/TestProjects.kt

@@ -13,5 +13,6 @@ object TestProjects {
     const val javaLogger = "application/javaLogger"
     const val macOptions = "application/macOptions"
     const val optionsWithSpaces = "application/optionsWithSpaces"
+    const val unpackSkiko = "application/unpackSkiko"
     const val jsMpp = "misc/jsMpp"
 }

gradle-plugins/compose/src/test/kotlin/org/jetbrains/compose/test/assertUtils.kt

@@ -21,6 +21,9 @@ internal fun BuildResult.checks(fn: (BuildResultChecks) -> Unit) {
 }
 
 internal class BuildResultChecks(private val result: BuildResult) {
+    val log: String
+        get() = result.output
+
     fun logContains(substring: String) {
         if (!result.output.contains(substring)) {
             throw AssertionError("Test output does not contain the expected string: '$substring'")

gradle-plugins/compose/src/test/test-projects/application/unpackSkiko/build.gradle

@@ -0,0 +1,32 @@
+import org.jetbrains.compose.desktop.application.dsl.TargetFormat
+
+plugins {
+    id "org.jetbrains.kotlin.jvm"
+    id "org.jetbrains.compose"
+}
+
+repositories {
+    google()
+    mavenCentral()
+    jcenter()
+    maven {
+        url "https://maven.pkg.jetbrains.space/public/p/compose/dev"
+    }
+}
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib"
+    implementation compose.desktop.currentOs
+}
+
+compose.desktop {
+    application {
+        mainClass = "MainKt"
+        nativeDistributions {
+            targetFormats(TargetFormat.Dmg, TargetFormat.Msi, TargetFormat.Deb)
+
+            packageVersion = "1.0.0"
+            packageName = "TestPackage"
+        }
+    }
+}

gradle-plugins/compose/src/test/test-projects/application/unpackSkiko/settings.gradle

@@ -0,0 +1,11 @@
+pluginManagement {
+    plugins {
+        id 'org.jetbrains.kotlin.jvm' version 'KOTLIN_VERSION_PLACEHOLDER'
+        id 'org.jetbrains.compose' version 'COMPOSE_VERSION_PLACEHOLDER'
+    }
+    repositories {
+        mavenLocal()
+        gradlePluginPortal()
+    }
+}
+rootProject.name = "simple"

gradle-plugins/compose/src/test/test-projects/application/unpackSkiko/src/main/kotlin/main.kt

@@ -0,0 +1,4 @@
+fun main() {
+    val skikoPath = System.getProperty("skiko.library.path")
+    System.out.println("Read skiko library path: '$skikoPath'")
+}