From 2f0af5d2594a78cd5610aee58511abc6129d7f9c Mon Sep 17 00:00:00 2001 From: cfig Date: Mon, 29 May 2023 12:45:10 +0800 Subject: [PATCH] Issue #117: respect original check flags for ramdisk compression ramdisk xz check flags: crc32/crc64 etc. XiaoMi recovery image uses CRC32 and can not suppport CRC64 --- bbootimg/src/main/kotlin/bootimg/Common.kt | 4 +- bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt | 20 +++- .../main/kotlin/bootimg/v2/BootV2Dialects.kt | 4 +- bbootimg/src/main/kotlin/bootimg/v3/BootV3.kt | 16 ++- .../src/main/kotlin/bootimg/v3/VendorBoot.kt | 19 +++- .../src/main/kotlin/ota/DeltaGenerator.kt | 2 +- .../src/main/kotlin/cfig/helper/ZipHelper.kt | 103 ++++++++++++++++-- integrationTest.py | 2 + src/integrationTest/resources_2 | 2 +- 9 files changed, 149 insertions(+), 23 deletions(-) diff --git a/bbootimg/src/main/kotlin/bootimg/Common.kt b/bbootimg/src/main/kotlin/bootimg/Common.kt index aa475c8..bbeb3ec 100644 --- a/bbootimg/src/main/kotlin/bootimg/Common.kt +++ b/bbootimg/src/main/kotlin/bootimg/Common.kt @@ -258,7 +258,7 @@ class Common { } //using preset fs_config - fun packRootfs(rootDir: String, ramdiskGz: String) { + fun packRootfs(rootDir: String, ramdiskGz: String, compressorArgs: String? = null) { val root = File(rootDir).path log.info("Packing rootfs $root ...") when { @@ -280,7 +280,7 @@ class Common { ramdiskGz.endsWith(".xz") -> { val f = ramdiskGz.removeSuffix(".xz") AndroidCpio().pack(root, f, "${f}_filelist.txt") - FileInputStream(f).use { ZipHelper.xz(ramdiskGz, it) } + FileInputStream(f).use { ZipHelper.xz(ramdiskGz, it, compressorArgs!!) } } ramdiskGz.endsWith(".cpio") -> { val f = ramdiskGz.removeSuffix(".cpio") diff --git a/bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt b/bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt index ca49556..5885e4c 100644 --- a/bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt +++ b/bbootimg/src/main/kotlin/bootimg/v2/BootV2.kt @@ -16,11 +16,13 @@ package cfig.bootimg.v2 import avb.AVBInfo import cfig.Avb +import cfig.bootimg.Common as C import cfig.bootimg.Common import cfig.bootimg.Common.Companion.deleleIfExists import cfig.bootimg.Signer import cfig.helper.Dumpling import cfig.helper.Helper +import cfig.helper.ZipHelper import cfig.packable.VBMetaParser import cfig.utils.DTC import cfig.utils.EnvironmentVerifier @@ -39,7 +41,7 @@ import java.nio.ByteOrder data class BootV2( var info: MiscInfo = MiscInfo(), var kernel: CommArgs = CommArgs(), - var ramdisk: CommArgs = CommArgs(), + var ramdisk: RamdiskArgs = RamdiskArgs(), var secondBootloader: CommArgs? = null, var recoveryDtbo: CommArgsLong? = null, var dtb: CommArgsLong? = null, @@ -68,6 +70,14 @@ data class BootV2( var loadOffset: Long = 0, ) + data class RamdiskArgs( + var file: String? = null, + var position: Long = 0, + var size: Int = 0, + var loadOffset: Long = 0, + var xzFlags: String? = null + ) + data class CommArgsLong( var file: String? = null, var position: Long = 0, @@ -193,10 +203,14 @@ data class BootV2( Common.dumpKernel(Helper.Slice(info.output, kernel.position.toInt(), kernel.size, kernel.file!!)) //ramdisk if (this.ramdisk.size > 0) { - val fmt = Common.dumpRamdisk( + val fmt = C.dumpRamdisk( Helper.Slice(info.output, ramdisk.position.toInt(), ramdisk.size, ramdisk.file!!), "${workDir}root" ) this.ramdisk.file = this.ramdisk.file!! + ".$fmt" + if (fmt == "xz") { + val checkType = ZipHelper.xzStreamFlagCheckTypeToString(ZipHelper.parseStreamFlagCheckType(this.ramdisk.file!!)) + this.ramdisk.xzFlags = checkType + } //dump info again mapper.writerWithDefaultPrettyPrinter().writeValue(File(workDir + this.info.json), this) } @@ -398,7 +412,7 @@ data class BootV2( File(this.ramdisk.file!!).deleleIfExists() File(this.ramdisk.file!!.removeSuffix(".gz")).deleleIfExists() //Common.packRootfs("${workDir}/root", this.ramdisk.file!!, Common.parseOsMajor(info.osVersion.toString())) - Common.packRootfs("${workDir}/root", this.ramdisk.file!!) + Common.packRootfs("${workDir}/root", this.ramdisk.file!!, this.ramdisk.xzFlags) } this.ramdisk.size = File(this.ramdisk.file!!).length().toInt() } diff --git a/bbootimg/src/main/kotlin/bootimg/v2/BootV2Dialects.kt b/bbootimg/src/main/kotlin/bootimg/v2/BootV2Dialects.kt index aa6ead0..2972de8 100644 --- a/bbootimg/src/main/kotlin/bootimg/v2/BootV2Dialects.kt +++ b/bbootimg/src/main/kotlin/bootimg/v2/BootV2Dialects.kt @@ -37,7 +37,7 @@ import java.nio.ByteOrder data class BootV2Dialects( var info: MiscInfo = MiscInfo(), var kernel: CommArgs = CommArgs(), - var ramdisk: CommArgs = CommArgs(), + var ramdisk: BootV2.RamdiskArgs = BootV2.RamdiskArgs(), var secondBootloader: CommArgs? = null, var recoveryDtbo: CommArgsLong? = null, var dtb: CommArgsLong? = null, @@ -378,7 +378,7 @@ data class BootV2Dialects( File(this.ramdisk.file!!).deleleIfExists() File(this.ramdisk.file!!.removeSuffix(".gz")).deleleIfExists() //Common.packRootfs("${workDir}/root", this.ramdisk.file!!, Common.parseOsMajor(info.osVersion.toString())) - Common.packRootfs("${workDir}/root", this.ramdisk.file!!) + Common.packRootfs("${workDir}/root", this.ramdisk.file!!, this.ramdisk.xzFlags) } this.ramdisk.size = File(this.ramdisk.file!!).length().toInt() } diff --git a/bbootimg/src/main/kotlin/bootimg/v3/BootV3.kt b/bbootimg/src/main/kotlin/bootimg/v3/BootV3.kt index 7bc62b1..66ef403 100644 --- a/bbootimg/src/main/kotlin/bootimg/v3/BootV3.kt +++ b/bbootimg/src/main/kotlin/bootimg/v3/BootV3.kt @@ -25,6 +25,7 @@ import cfig.bootimg.Common.Companion.getPaddingSize import cfig.bootimg.Signer import cfig.helper.Helper import cfig.helper.Dumpling +import cfig.helper.ZipHelper import cfig.packable.VBMetaParser import com.fasterxml.jackson.databind.ObjectMapper import de.vandermeer.asciitable.AsciiTable @@ -41,7 +42,7 @@ import cfig.bootimg.Common as C data class BootV3( var info: MiscInfo = MiscInfo(), var kernel: CommArgs = CommArgs(), - val ramdisk: CommArgs = CommArgs(), + val ramdisk: RamdiskArgs = RamdiskArgs(), var bootSignature: CommArgs = CommArgs(), ) { companion object { @@ -105,6 +106,13 @@ data class BootV3( var size: Int = 0, ) + data class RamdiskArgs ( + var file: String = "", + var position: Int = 0, + var size: Int = 0, + var xzFlags: String? = null + ) + fun pack(): BootV3 { if (this.kernel.size > 0) { this.kernel.size = File(this.kernel.file).length().toInt() @@ -119,7 +127,7 @@ data class BootV3( //TODO: remove cpio in C/C++ //C.packRootfs("$workDir/root", this.ramdisk.file, C.parseOsMajor(info.osVersion)) // enable advance JAVA cpio - C.packRootfs("$workDir/root", this.ramdisk.file) + C.packRootfs("$workDir/root", this.ramdisk.file, this.ramdisk.xzFlags) } this.ramdisk.size = File(this.ramdisk.file).length().toInt() } @@ -232,6 +240,10 @@ data class BootV3( Helper.Slice(info.output, ramdisk.position, ramdisk.size, ramdisk.file), "${workDir}root" ) this.ramdisk.file = this.ramdisk.file + ".$fmt" + if (fmt == "xz") { + val checkType = ZipHelper.xzStreamFlagCheckTypeToString(ZipHelper.parseStreamFlagCheckType(this.ramdisk.file)) + this.ramdisk.xzFlags = checkType + } } //bootsig diff --git a/bbootimg/src/main/kotlin/bootimg/v3/VendorBoot.kt b/bbootimg/src/main/kotlin/bootimg/v3/VendorBoot.kt index 4c06817..309153d 100644 --- a/bbootimg/src/main/kotlin/bootimg/v3/VendorBoot.kt +++ b/bbootimg/src/main/kotlin/bootimg/v3/VendorBoot.kt @@ -22,6 +22,7 @@ import cfig.bootimg.Common.Companion.deleleIfExists import cfig.bootimg.Signer import cfig.helper.Dumpling import cfig.helper.Helper +import cfig.helper.ZipHelper import cfig.packable.VBMetaParser import cfig.utils.DTC import cfig.utils.EnvironmentVerifier @@ -37,7 +38,7 @@ import cfig.bootimg.Common as C data class VendorBoot( var info: MiscInfo = MiscInfo(), - var ramdisk: CommArgs = CommArgs(), + var ramdisk: RamdiskArgs = RamdiskArgs(), var dtb: CommArgs = CommArgs(), var ramdisk_table: Vrt = Vrt(), var bootconfig: CommArgs = CommArgs(), @@ -49,6 +50,14 @@ data class VendorBoot( var loadAddr: Long = 0, ) + data class RamdiskArgs( + var file: String = "", + var position: Long = 0, + var size: Int = 0, + var loadAddr: Long = 0, + var xzFlags: String? = null + ) + data class MiscInfo( var output: String = "", var json: String = "", @@ -227,7 +236,7 @@ data class VendorBoot( //Fixed: remove cpio in C/C++ //C.packRootfs("$workDir/root", this.ramdisk.file, parseOsMajor()) //enable advance JAVA cpio - C.packRootfs("$workDir/root", this.ramdisk.file) + C.packRootfs("$workDir/root", this.ramdisk.file, this.ramdisk.xzFlags) } this.ramdisk.size = File(this.ramdisk.file).length().toInt() } @@ -235,7 +244,7 @@ data class VendorBoot( this.ramdisk_table.ramdidks.forEachIndexed { index, it -> File(it.file).deleleIfExists() log.info(workDir + "root.${index + 1} -> " + it.file) - C.packRootfs(workDir + "root.${index + 1}", it.file) + C.packRootfs(workDir + "root.${index + 1}", it.file, this.ramdisk.xzFlags) } this.ramdisk.size = this.ramdisk_table.ramdidks.sumOf { File(it.file).length() }.toInt() } @@ -345,6 +354,10 @@ data class VendorBoot( this.ramdisk_table.ramdidks.isEmpty()) //@formatter:on this.ramdisk.file = this.ramdisk.file + ".$fmt" + if (fmt == "xz") { + val checkType = ZipHelper.xzStreamFlagCheckTypeToString(ZipHelper.parseStreamFlagCheckType(this.ramdisk.file)) + this.ramdisk.xzFlags = checkType + } //dtb C.dumpDtb(Helper.Slice(info.output, dtb.position.toInt(), dtb.size, dtb.file)) //vrt diff --git a/bbootimg/src/main/kotlin/ota/DeltaGenerator.kt b/bbootimg/src/main/kotlin/ota/DeltaGenerator.kt index ad61a9d..d5623bf 100644 --- a/bbootimg/src/main/kotlin/ota/DeltaGenerator.kt +++ b/bbootimg/src/main/kotlin/ota/DeltaGenerator.kt @@ -69,7 +69,7 @@ class DeltaGenerator { //try xz File.createTempFile("pre", "suf").let { tempFile -> tempFile.deleteOnExit() - ZipHelper.xz(tempFile.absolutePath, ByteArrayInputStream(inData)) + ZipHelper.xz(tempFile.absolutePath, ByteArrayInputStream(inData), "CRC64") log.debug("raw=${inData.size}, xz=" + tempFile.length()) if (bestSize > tempFile.length()) { bestType = Type.REPLACE_XZ diff --git a/helper/src/main/kotlin/cfig/helper/ZipHelper.kt b/helper/src/main/kotlin/cfig/helper/ZipHelper.kt index 68c5ab6..8972bba 100644 --- a/helper/src/main/kotlin/cfig/helper/ZipHelper.kt +++ b/helper/src/main/kotlin/cfig/helper/ZipHelper.kt @@ -17,11 +17,8 @@ package cfig.helper import cc.cfig.io.Struct import cfig.helper.Helper.Companion.check_call import cfig.helper.Helper.Companion.check_output -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream -import org.apache.commons.compress.archivers.zip.ZipFile -import org.apache.commons.compress.archivers.zip.ZipMethod +import org.apache.commons.compress.archivers.zip.* +import org.apache.commons.compress.compressors.CompressorOutputStream import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream @@ -34,9 +31,10 @@ import org.apache.commons.exec.CommandLine import org.apache.commons.exec.DefaultExecutor import org.apache.commons.exec.PumpStreamHandler import org.slf4j.LoggerFactory +import org.tukaani.xz.LZMA2Options import org.tukaani.xz.XZFormatException +import org.tukaani.xz.XZOutputStream import java.io.* -import java.lang.RuntimeException import java.net.URI import java.nio.file.FileSystems import java.nio.file.Files @@ -50,6 +48,43 @@ import kotlin.reflect.jvm.isAccessible class ZipHelper { class ZipEntryRecipe(val data: ByteArray, val name: String, val method: ZipMethod) + class XZCompressorOutputStream2 : CompressorOutputStream { + private val out: XZOutputStream + + constructor(outputStream: OutputStream?) { + out = XZOutputStream(outputStream, LZMA2Options()) + } + + constructor(outputStream: OutputStream?, checkType: Int) { + out = XZOutputStream(outputStream, LZMA2Options(), checkType) + } + + @Throws(IOException::class) + override fun write(b: Int) { + out.write(b) + } + + @Throws(IOException::class) + override fun write(buf: ByteArray, off: Int, len: Int) { + out.write(buf, off, len) + } + + @Throws(IOException::class) + override fun flush() { + out.flush() + } + + @Throws(IOException::class) + fun finish() { + out.finish() + } + + @Throws(IOException::class) + override fun close() { + out.close() + } + } + companion object { private val log = LoggerFactory.getLogger("ZipHelper") @@ -86,9 +121,11 @@ class ZipHelper { entryOut.mkdir() log.debug("Unzipping[d]: ${entry.name}") } + entry.isUnixSymlink -> { throw IllegalArgumentException("this should not happen: Found dir ${entry.name}") } + else -> { val entryOut = File(outDir + "/" + entry.name) log.debug("Unzipping[f]: ${entry.name}") @@ -282,6 +319,54 @@ class ZipHelper { log.info("decompress(lzma) done: $compressedFile -> $decompressedFile") } + // https://tukaani.org/xz/xz-file-format.txt + // 2.1.1. Stream Header + fun parseStreamFlagCheckType(file: String): Int { + FileInputStream(file).use { fis -> + val ba = ByteArray(6) + check(fis.read(ba) == ba.size) + check(ba.contentEquals(byteArrayOf(0xfd.toByte(), 0x37, 0x7a, 0x58, 0x5a, 0x00))) { + log.warn("wrong magic bytes in xz header") + } + check(fis.read(ba) == ba.size) + check(ba[0] == 0x00.toByte()) + when (ba[1].toInt()) { + 0x00 -> log.info("NONE") + 0x01 -> log.info("CRC32") + 0x04 -> log.info("CRC64") + 0x0a -> log.info("SHA256") + else -> throw IllegalArgumentException( + "unsupported StreamFlag.CheckType: 0x" + ba[1].toInt().toString(16) + ) + } + return ba[1].toInt() + } + } + + fun xzStreamFlagCheckTypeToString(type: Int): String { + return when (type) { + 0x00 -> "NONE" + 0x01 -> "CRC32" + 0x04 -> "CRC64" + 0x0a -> "SHA256" + else -> throw IllegalArgumentException( + "unsupported StreamFlag.CheckType: 0x" + type.toString(16) + ) + } + } + + fun xzStreamFlagCheckTypeFromString(typeStr: String): Int { + return when (typeStr) { + "NONE" -> 0x00 + "CRC32" -> 0x01 + "CRC64" -> 0x04 + "SHA256" -> 0x0a + else -> throw IllegalArgumentException( + "unsupported StreamFlag.CheckType: 0x$typeStr" + ) + } + } + fun isXz(compressedFile: String): Boolean { return try { FileInputStream(compressedFile).use { fis -> @@ -294,10 +379,10 @@ class ZipHelper { } } - fun xz(compressedFile: String, fis: InputStream) { - log.info("Compress(xz) ... ") + fun xz(compressedFile: String, fis: InputStream, checkType: String) { + log.info("Compress(xz), with checkType $checkType... ") FileOutputStream(compressedFile).use { fos -> - XZCompressorOutputStream(fos).use { gos -> + XZCompressorOutputStream2(fos, ZipHelper.xzStreamFlagCheckTypeFromString(checkType)).use { gos -> val buffer = ByteArray(1024) while (true) { val bytesRead = fis.read(buffer) diff --git a/integrationTest.py b/integrationTest.py index f590b47..ea5dbb8 100755 --- a/integrationTest.py +++ b/integrationTest.py @@ -176,6 +176,8 @@ def main(): verifySingleDir(resDir2, "issue_91_unsigned_vendor_boot") # Issue 109: vendor_boot w/o dtb verifySingleDir(resDir2, "issue_109_vendor_boot_no_dtb") + # Issue 117: xz crc32/crc64 + verifySingleDir(resDir2, "issue_117_xz_crc") log.info(successLogo) diff --git a/src/integrationTest/resources_2 b/src/integrationTest/resources_2 index 0caca00..bf9b8b9 160000 --- a/src/integrationTest/resources_2 +++ b/src/integrationTest/resources_2 @@ -1 +1 @@ -Subproject commit 0caca0031646be47eb89ae1e4380a524f63ec52b +Subproject commit bf9b8b996723536fcc19768185d87272f5363f76