cfig
7 months ago
parent
dafd084134
commit
8e6ba4662f
@ -0,0 +1,256 @@
|
|||||||
|
package cfig.lazybox
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
import java.text.SimpleDateFormat
|
||||||
|
import java.util.Date
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
import kotlin.system.exitProcess
|
||||||
|
|
||||||
|
private object Colors {
|
||||||
|
const val RED = "\u001B[0;31m"
|
||||||
|
const val GREEN = "\u001B[0;32m"
|
||||||
|
const val YELLOW = "\u001B[1;33m"
|
||||||
|
const val BLUE = "\u001B[0;34m"
|
||||||
|
const val CYAN = "\u001B[0;36m"
|
||||||
|
const val NC = "\u001B[0m"
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun printHeader(message: String) {
|
||||||
|
println("${Colors.BLUE}========================================${Colors.NC}")
|
||||||
|
println("${Colors.BLUE}$message${Colors.NC}")
|
||||||
|
println("${Colors.BLUE}========================================${Colors.NC}")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun printInfo(message: String) {
|
||||||
|
println("${Colors.GREEN}[INFO]${Colors.NC} $message")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun printWarn(message: String) {
|
||||||
|
println("${Colors.YELLOW}[WARN]${Colors.NC} $message")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun printError(message: String) {
|
||||||
|
println("${Colors.RED}[ERROR]${Colors.NC} $message")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun printSuccess(message: String) {
|
||||||
|
println("${Colors.GREEN}[OK]${Colors.NC} $message")
|
||||||
|
}
|
||||||
|
|
||||||
|
private data class CommandResult(val output: List<String>, val exitCode: Int)
|
||||||
|
|
||||||
|
private fun runHostCommand(vararg command: String): CommandResult {
|
||||||
|
return try {
|
||||||
|
val process = ProcessBuilder(*command)
|
||||||
|
.redirectErrorStream(true)
|
||||||
|
.start()
|
||||||
|
val output = process.inputStream.bufferedReader().readLines()
|
||||||
|
if (!process.waitFor(10, TimeUnit.SECONDS)) {
|
||||||
|
process.destroy()
|
||||||
|
CommandResult(output, 124)
|
||||||
|
} else {
|
||||||
|
CommandResult(output, process.exitValue())
|
||||||
|
}
|
||||||
|
} catch (e: Exception) {
|
||||||
|
CommandResult(emptyList(), 127)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class AdbHelper {
|
||||||
|
fun isAdbAvailable(): Boolean {
|
||||||
|
return runHostCommand("adb", "version").exitCode == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
fun isConnected(): Boolean {
|
||||||
|
val devices = runHostCommand("adb", "devices").output
|
||||||
|
return devices.any { line ->
|
||||||
|
val trimmed = line.trim()
|
||||||
|
trimmed.isNotEmpty() && trimmed.endsWith("device") && !trimmed.startsWith("List")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun executeOnDevice(command: String): List<String> {
|
||||||
|
return runHostCommand("adb", "shell", command).output
|
||||||
|
}
|
||||||
|
|
||||||
|
fun findApexFiles(): List<String> {
|
||||||
|
val apexFiles = mutableListOf<String>()
|
||||||
|
apexFiles.addAll(executeOnDevice("find / -name '*.apex' 2>/dev/null"))
|
||||||
|
apexFiles.addAll(executeOnDevice("find / -name '*.capex' 2>/dev/null"))
|
||||||
|
return apexFiles.filter { it.isNotBlank() }
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getApexMounts(): List<String> {
|
||||||
|
return executeOnDevice("mount | grep apex").filter { it.isNotBlank() }
|
||||||
|
}
|
||||||
|
|
||||||
|
fun findPermissionController(): List<String> {
|
||||||
|
return executeOnDevice("find / -name '*permission*.apex' 2>/dev/null").filter { it.isNotBlank() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class Apex {
|
||||||
|
private val adb = AdbHelper()
|
||||||
|
|
||||||
|
private fun checkConnection() {
|
||||||
|
printHeader("Check ADB connection")
|
||||||
|
|
||||||
|
if (!adb.isAdbAvailable()) {
|
||||||
|
printError("ADB is not installed or not in PATH")
|
||||||
|
exitProcess(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!adb.isConnected()) {
|
||||||
|
printError("No connected devices found")
|
||||||
|
println("Please ensure:")
|
||||||
|
println(" 1. Device connected via USB")
|
||||||
|
println(" 2. USB debugging enabled")
|
||||||
|
println(" 3. ADB authorization granted")
|
||||||
|
exitProcess(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
printSuccess("ADB connection OK")
|
||||||
|
runHostCommand("adb", "devices").output.forEach { println(it) }
|
||||||
|
println()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun findApexFiles() {
|
||||||
|
printHeader("Search APEX files")
|
||||||
|
|
||||||
|
printInfo("Searching APEX files on device...")
|
||||||
|
val apexFiles = adb.findApexFiles()
|
||||||
|
|
||||||
|
if (apexFiles.isEmpty()) {
|
||||||
|
printWarn("No APEX files found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
printSuccess("Found ${apexFiles.size} APEX files")
|
||||||
|
println()
|
||||||
|
|
||||||
|
printHeader("APEX file list")
|
||||||
|
apexFiles.sorted().forEachIndexed { index, file ->
|
||||||
|
println("${index + 1}. $file")
|
||||||
|
}
|
||||||
|
println()
|
||||||
|
|
||||||
|
val apexCount = apexFiles.count { it.endsWith(".apex") }
|
||||||
|
val capexCount = apexFiles.count { it.endsWith(".capex") }
|
||||||
|
|
||||||
|
printHeader("Statistics")
|
||||||
|
println("Total: ${apexFiles.size}")
|
||||||
|
println(" - .apex files: $apexCount")
|
||||||
|
println(" - .capex files: $capexCount")
|
||||||
|
println()
|
||||||
|
|
||||||
|
printHeader("Group by location")
|
||||||
|
println()
|
||||||
|
|
||||||
|
val byLocation = apexFiles.groupBy { file ->
|
||||||
|
when {
|
||||||
|
file.contains("/system/apex") -> "/system/apex"
|
||||||
|
file.startsWith("/apex") -> "/apex"
|
||||||
|
file.contains("/system/priv-app") -> "/system/priv-app"
|
||||||
|
file.contains("/data") -> "/data"
|
||||||
|
else -> "other"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
byLocation.forEach { (location, files) ->
|
||||||
|
println("${Colors.CYAN}$location:${Colors.NC}")
|
||||||
|
files.sorted().forEach { println(" $it") }
|
||||||
|
println()
|
||||||
|
}
|
||||||
|
|
||||||
|
val timeStamp = SimpleDateFormat("yyyyMMdd_HHmmss").format(Date())
|
||||||
|
val outputFile = File("apex_files_$timeStamp.txt")
|
||||||
|
outputFile.writeText(apexFiles.sorted().joinToString("\n"))
|
||||||
|
printInfo("Saved to: ${outputFile.absolutePath}")
|
||||||
|
println()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun checkMounts() {
|
||||||
|
printHeader("Check APEX mounts")
|
||||||
|
|
||||||
|
val mounts = adb.getApexMounts()
|
||||||
|
if (mounts.isEmpty()) {
|
||||||
|
printWarn("No mounted APEX found")
|
||||||
|
} else {
|
||||||
|
mounts.forEach { println(it) }
|
||||||
|
}
|
||||||
|
println()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun checkPermissionController() {
|
||||||
|
printHeader("Check PermissionController")
|
||||||
|
|
||||||
|
printInfo("Searching permission-related APEX...")
|
||||||
|
val permApex = adb.findPermissionController()
|
||||||
|
if (permApex.isEmpty()) {
|
||||||
|
printWarn("No permission-related APEX found")
|
||||||
|
} else {
|
||||||
|
permApex.forEach { println(it) }
|
||||||
|
}
|
||||||
|
println()
|
||||||
|
|
||||||
|
printInfo("Searching permission XML files...")
|
||||||
|
val privappPerms = adb.executeOnDevice("find / -name 'privapp-permissions*.xml' 2>/dev/null")
|
||||||
|
.filter { it.isNotBlank() }
|
||||||
|
if (privappPerms.isEmpty()) {
|
||||||
|
printWarn("No permission XML files found")
|
||||||
|
} else {
|
||||||
|
privappPerms.forEach { println(it) }
|
||||||
|
}
|
||||||
|
println()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun showHelp() {
|
||||||
|
println(
|
||||||
|
"""
|
||||||
|
Usage: lazybox apex [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-h, --help Show help
|
||||||
|
-a, --all Run all checks
|
||||||
|
-f, --find Find APEX files (default)
|
||||||
|
-m, --mounts Check APEX mounts
|
||||||
|
-p, --perm Check PermissionController
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
lazybox apex
|
||||||
|
lazybox apex --all
|
||||||
|
lazybox apex --mounts
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun run(args: Array<String>) {
|
||||||
|
when {
|
||||||
|
args.isEmpty() || args[0] == "-f" || args[0] == "--find" -> {
|
||||||
|
checkConnection()
|
||||||
|
findApexFiles()
|
||||||
|
}
|
||||||
|
args[0] == "-h" || args[0] == "--help" -> {
|
||||||
|
showHelp()
|
||||||
|
}
|
||||||
|
args[0] == "-a" || args[0] == "--all" -> {
|
||||||
|
checkConnection()
|
||||||
|
findApexFiles()
|
||||||
|
checkMounts()
|
||||||
|
checkPermissionController()
|
||||||
|
}
|
||||||
|
args[0] == "-m" || args[0] == "--mounts" -> {
|
||||||
|
checkConnection()
|
||||||
|
checkMounts()
|
||||||
|
}
|
||||||
|
args[0] == "-p" || args[0] == "--perm" -> {
|
||||||
|
checkConnection()
|
||||||
|
checkPermissionController()
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
println("Unknown option: ${args[0]}")
|
||||||
|
showHelp()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,147 @@
|
|||||||
|
package cfig.lazybox
|
||||||
|
|
||||||
|
import org.slf4j.LoggerFactory
|
||||||
|
import java.io.File
|
||||||
|
|
||||||
|
object BootTimeABCompare {
|
||||||
|
private val log = LoggerFactory.getLogger(BootTimeABCompare::class.java)
|
||||||
|
private val nameSanitizer = Regex("[^A-Za-z_]")
|
||||||
|
|
||||||
|
fun run(args: Array<String>) {
|
||||||
|
if (args.isEmpty() || args.contains("--help") || args.contains("-h")) {
|
||||||
|
printUsage()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
val positionals = mutableListOf<String>()
|
||||||
|
val milestones = mutableListOf<String>()
|
||||||
|
var aName: String? = null
|
||||||
|
var bName: String? = null
|
||||||
|
var outDir: String? = null
|
||||||
|
var reportName: String? = null
|
||||||
|
|
||||||
|
var i = 0
|
||||||
|
while (i < args.size) {
|
||||||
|
val arg = args[i]
|
||||||
|
if (!arg.startsWith("--")) {
|
||||||
|
positionals.add(arg)
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fun requireValue(): String {
|
||||||
|
if (i + 1 >= args.size) {
|
||||||
|
throw IllegalArgumentException("Missing value for $arg")
|
||||||
|
}
|
||||||
|
return args[i + 1]
|
||||||
|
}
|
||||||
|
|
||||||
|
when (arg) {
|
||||||
|
"--a-name" -> {
|
||||||
|
aName = requireValue()
|
||||||
|
i += 2
|
||||||
|
}
|
||||||
|
"--b-name" -> {
|
||||||
|
bName = requireValue()
|
||||||
|
i += 2
|
||||||
|
}
|
||||||
|
"--out-dir" -> {
|
||||||
|
outDir = requireValue()
|
||||||
|
i += 2
|
||||||
|
}
|
||||||
|
"--report-name" -> {
|
||||||
|
reportName = requireValue()
|
||||||
|
i += 2
|
||||||
|
}
|
||||||
|
"--milestone" -> {
|
||||||
|
milestones.add(requireValue())
|
||||||
|
i += 2
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
throw IllegalArgumentException("Unknown option: $arg")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (positionals.size < 2) {
|
||||||
|
log.error("Missing required directories for A and B.")
|
||||||
|
printUsage()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
val aDir = File(positionals[0]).absoluteFile
|
||||||
|
val bDir = File(positionals[1]).absoluteFile
|
||||||
|
if (!aDir.exists() || !aDir.isDirectory) {
|
||||||
|
log.error("Directory not found: ${aDir.absolutePath}")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (!bDir.exists() || !bDir.isDirectory) {
|
||||||
|
log.error("Directory not found: ${bDir.absolutePath}")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
val script = findScript(File(".").absoluteFile)
|
||||||
|
if (script == null) {
|
||||||
|
log.error("boot_analysis.py not found under current or parent directories")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
val aLabel = aName ?: aDir.name
|
||||||
|
val bLabel = bName ?: bDir.name
|
||||||
|
val defaultOutDir = "${sanitizeName(aLabel)}_VS_${sanitizeName(bLabel)}"
|
||||||
|
|
||||||
|
val cmd = mutableListOf(
|
||||||
|
"python3",
|
||||||
|
script.absolutePath,
|
||||||
|
"--a-name",
|
||||||
|
aLabel,
|
||||||
|
"--a-dir",
|
||||||
|
aDir.absolutePath,
|
||||||
|
"--b-name",
|
||||||
|
bLabel,
|
||||||
|
"--b-dir",
|
||||||
|
bDir.absolutePath,
|
||||||
|
)
|
||||||
|
cmd.addAll(listOf("--out-dir", outDir ?: defaultOutDir))
|
||||||
|
if (reportName != null) {
|
||||||
|
cmd.addAll(listOf("--report-name", reportName!!))
|
||||||
|
}
|
||||||
|
if (milestones.isNotEmpty()) {
|
||||||
|
milestones.forEach { milestone ->
|
||||||
|
cmd.addAll(listOf("--milestone", milestone))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("Running boot_analysis.py for A=$aLabel B=$bLabel")
|
||||||
|
val process = ProcessBuilder(cmd)
|
||||||
|
.redirectOutput(ProcessBuilder.Redirect.INHERIT)
|
||||||
|
.redirectError(ProcessBuilder.Redirect.INHERIT)
|
||||||
|
.start()
|
||||||
|
|
||||||
|
val exit = process.waitFor()
|
||||||
|
if (exit != 0) {
|
||||||
|
log.error("boot_analysis.py failed with exit code: $exit")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun printUsage() {
|
||||||
|
println("Usage: stat AB <a_dir> <b_dir> [--a-name NAME] [--b-name NAME]")
|
||||||
|
println(" [--out-dir DIR] [--report-name NAME] [--milestone NAME]...")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun findScript(startDir: File): File? {
|
||||||
|
var dir: File? = startDir
|
||||||
|
while (dir != null) {
|
||||||
|
val candidate = File(dir, "boot_analysis.py")
|
||||||
|
if (candidate.exists()) {
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
dir = dir.parentFile
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun sanitizeName(name: String): String {
|
||||||
|
return nameSanitizer.replace(name, "_")
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,341 @@
|
|||||||
|
package cfig.lazybox
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
import kotlin.math.sqrt
|
||||||
|
import kotlin.system.exitProcess
|
||||||
|
|
||||||
|
data class BootRun(val name: String, val stages: Map<String, Double>)
|
||||||
|
|
||||||
|
data class StageStats(
|
||||||
|
val mean: Double,
|
||||||
|
val std: Double,
|
||||||
|
val min: Double,
|
||||||
|
val max: Double,
|
||||||
|
val jitter: Double,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class Outlier(val run: String, val stage: String, val value: Double, val median: Double, val threshold: Double)
|
||||||
|
|
||||||
|
object BootTimeStability {
|
||||||
|
private val requiredStages = listOf(
|
||||||
|
"Kernel Start",
|
||||||
|
"Boot Progress Start",
|
||||||
|
"AMS Ready",
|
||||||
|
"Boot Completed",
|
||||||
|
"Launcher Start",
|
||||||
|
"Launcher loaded",
|
||||||
|
)
|
||||||
|
|
||||||
|
private fun parseSeconds(token: String): Double? {
|
||||||
|
val t = token.trim().removeSuffix("s").trim()
|
||||||
|
return t.toDoubleOrNull()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseMergedBootReport(file: File): Map<String, Double> {
|
||||||
|
val found = mutableMapOf<String, Double>()
|
||||||
|
|
||||||
|
file.forEachLine { line ->
|
||||||
|
val trimmed = line.trim()
|
||||||
|
if (!trimmed.startsWith("|")) return@forEachLine
|
||||||
|
if (trimmed.startsWith("|---") || trimmed.contains("TimeStamp") && trimmed.contains("Delta(A)")) return@forEachLine
|
||||||
|
|
||||||
|
val cols = trimmed
|
||||||
|
.split("|")
|
||||||
|
.map { it.trim() }
|
||||||
|
.filter { it.isNotEmpty() }
|
||||||
|
|
||||||
|
if (cols.size < 6) return@forEachLine
|
||||||
|
|
||||||
|
val logEvent = cols[1]
|
||||||
|
val visualEvent = cols[2]
|
||||||
|
val deltaA = parseSeconds(cols[3]) ?: return@forEachLine
|
||||||
|
|
||||||
|
val logEventLc = logEvent.lowercase()
|
||||||
|
val visualEventLc = visualEvent.lowercase()
|
||||||
|
|
||||||
|
fun maybeSet(stage: String, matched: Boolean) {
|
||||||
|
if (matched && !found.containsKey(stage)) {
|
||||||
|
found[stage] = deltaA
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
maybeSet("Kernel Start", logEventLc == "kernel start" || visualEventLc == "kernel start")
|
||||||
|
maybeSet("Boot Progress Start", logEventLc == "boot progress start" || visualEventLc == "boot progress start")
|
||||||
|
maybeSet("AMS Ready", logEventLc == "ams ready" || visualEventLc == "ams ready")
|
||||||
|
maybeSet("Boot Completed", logEventLc == "boot completed" || visualEventLc == "boot completed")
|
||||||
|
maybeSet("Launcher Start", logEventLc == "launcher start" || visualEventLc == "launcher start")
|
||||||
|
maybeSet("Launcher loaded", logEventLc == "launcher loaded" || visualEventLc == "launcher loaded")
|
||||||
|
}
|
||||||
|
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun median(values: List<Double>): Double {
|
||||||
|
if (values.isEmpty()) return 0.0
|
||||||
|
val sorted = values.sorted()
|
||||||
|
val n = sorted.size
|
||||||
|
return if (n % 2 == 1) {
|
||||||
|
sorted[n / 2]
|
||||||
|
} else {
|
||||||
|
(sorted[n / 2 - 1] + sorted[n / 2]) / 2.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun mean(values: List<Double>): Double = if (values.isEmpty()) 0.0 else values.sum() / values.size
|
||||||
|
|
||||||
|
private fun sampleStd(values: List<Double>): Double {
|
||||||
|
if (values.size <= 1) return 0.0
|
||||||
|
val m = mean(values)
|
||||||
|
val variance = values.sumOf { (it - m) * (it - m) } / (values.size - 1)
|
||||||
|
return sqrt(variance)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun statsForStage(runs: List<BootRun>, stage: String): StageStats {
|
||||||
|
val values = runs.mapNotNull { it.stages[stage] }
|
||||||
|
if (values.isEmpty()) return StageStats(0.0, 0.0, 0.0, 0.0, 0.0)
|
||||||
|
val minV = values.minOrNull() ?: 0.0
|
||||||
|
val maxV = values.maxOrNull() ?: 0.0
|
||||||
|
return StageStats(
|
||||||
|
mean = mean(values),
|
||||||
|
std = sampleStd(values),
|
||||||
|
min = minV,
|
||||||
|
max = maxV,
|
||||||
|
jitter = maxV - minV,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun format3(v: Double): String = String.format("%.3f", v)
|
||||||
|
|
||||||
|
private fun markdownTable(runs: List<BootRun>): String {
|
||||||
|
val header = buildString {
|
||||||
|
append("| Run | ")
|
||||||
|
append(requiredStages.joinToString(" | "))
|
||||||
|
append(" |\n")
|
||||||
|
append("|:--|")
|
||||||
|
append(requiredStages.joinToString("|") { "--:" })
|
||||||
|
append("|\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
val rows = runs.joinToString("\n") { run ->
|
||||||
|
val cols = requiredStages.joinToString(" | ") { stage ->
|
||||||
|
val v = run.stages[stage]
|
||||||
|
if (v == null) "" else format3(v)
|
||||||
|
}
|
||||||
|
"| ${run.name} | $cols |"
|
||||||
|
}
|
||||||
|
|
||||||
|
return header + rows + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun markdownStatsTable(stats: Map<String, StageStats>): String {
|
||||||
|
fun s(stage: String) = stats[stage] ?: StageStats(0.0, 0.0, 0.0, 0.0, 0.0)
|
||||||
|
|
||||||
|
val header = "| Metric | ${requiredStages.joinToString(" | ")} |"
|
||||||
|
val sep = "|----------------|" + requiredStages.joinToString("|") { "------:" } + "|"
|
||||||
|
|
||||||
|
val lineMean = "| **Mean** | " + requiredStages.joinToString(" | ") { stage -> format3(s(stage).mean) } + " |"
|
||||||
|
val lineStd = "| **Std Dev** | " + requiredStages.joinToString(" | ") { stage -> format3(s(stage).std) } + " |"
|
||||||
|
val lineJitter = "| **Jitter** | " + requiredStages.joinToString(" | ") { stage -> format3(s(stage).jitter) } + " |"
|
||||||
|
|
||||||
|
return listOf(header, sep, lineMean, lineStd, lineJitter).joinToString("\n") + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
fun run(args: Array<String>) {
|
||||||
|
if (args.isEmpty()) {
|
||||||
|
System.err.println("Usage: lazybox stat outlier <directory>")
|
||||||
|
exitProcess(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
val inputDir = File(args[0])
|
||||||
|
if (!inputDir.exists() || !inputDir.isDirectory) {
|
||||||
|
System.err.println("Error: '${inputDir.absolutePath}' is not a directory")
|
||||||
|
exitProcess(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
val runDirs = inputDir.listFiles()?.filter { it.isDirectory }?.sortedBy { it.name } ?: emptyList()
|
||||||
|
|
||||||
|
println("[boot-stability] inputDir=${inputDir.absolutePath}")
|
||||||
|
println("[boot-stability] found ${runDirs.size} first-level subdirectories")
|
||||||
|
|
||||||
|
val parsedRuns = mutableListOf<BootRun>()
|
||||||
|
val skippedRuns = mutableListOf<String>()
|
||||||
|
|
||||||
|
for (dir in runDirs) {
|
||||||
|
val reportFile = File(dir, "merged_boot_report.md")
|
||||||
|
if (!reportFile.exists() || !reportFile.isFile) {
|
||||||
|
skippedRuns.add("${dir.name} (missing merged_boot_report.md)")
|
||||||
|
println("[boot-stability] SKIP ${dir.name}: merged_boot_report.md not found")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
val stageMap = parseMergedBootReport(reportFile)
|
||||||
|
|
||||||
|
val foundStages = requiredStages.filter { stageMap.containsKey(it) }
|
||||||
|
val missingStages = requiredStages.filter { !stageMap.containsKey(it) }
|
||||||
|
println(
|
||||||
|
"[boot-stability] RUN ${dir.name}: found=${foundStages.joinToString(", ")}" +
|
||||||
|
if (missingStages.isEmpty()) "" else "; missing=${missingStages.joinToString(", ")}"
|
||||||
|
)
|
||||||
|
parsedRuns.add(BootRun(dir.name, stageMap))
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsedRuns.isEmpty()) {
|
||||||
|
System.err.println("No valid runs found under: ${inputDir.absolutePath}")
|
||||||
|
exitProcess(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
val originalRuns = parsedRuns.toList()
|
||||||
|
|
||||||
|
println("[boot-stability] valid runs=${originalRuns.size}, skipped=${skippedRuns.size}")
|
||||||
|
|
||||||
|
val mediansByStage = requiredStages.associateWith { stage ->
|
||||||
|
median(originalRuns.mapNotNull { it.stages[stage] })
|
||||||
|
}
|
||||||
|
|
||||||
|
for (stage in requiredStages) {
|
||||||
|
val m = mediansByStage[stage] ?: 0.0
|
||||||
|
if (m > 0.0) {
|
||||||
|
println("[boot-stability] median[$stage]=${format3(m)} threshold=${format3(m * 2.0)}")
|
||||||
|
} else {
|
||||||
|
println("[boot-stability] median[$stage]=N/A (no samples)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val outliers = mutableListOf<Outlier>()
|
||||||
|
val outlierRunNames = mutableSetOf<String>()
|
||||||
|
|
||||||
|
for (stage in requiredStages) {
|
||||||
|
val medianV = mediansByStage[stage] ?: 0.0
|
||||||
|
val threshold = medianV * 2.0
|
||||||
|
for (run in originalRuns) {
|
||||||
|
val v = run.stages[stage] ?: continue
|
||||||
|
if (medianV > 0.0 && v > threshold) {
|
||||||
|
outliers.add(Outlier(run.name, stage, v, medianV, threshold))
|
||||||
|
outlierRunNames.add(run.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (outliers.isEmpty()) {
|
||||||
|
println("[boot-stability] outliers: none")
|
||||||
|
} else {
|
||||||
|
println("[boot-stability] outliers: ${outliers.size}")
|
||||||
|
outliers.forEach { o ->
|
||||||
|
println(
|
||||||
|
"[boot-stability] outlier run=${o.run} stage=${o.stage} " +
|
||||||
|
"value=${format3(o.value)} threshold=${format3(o.threshold)}"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
println("[boot-stability] outlier runs=${outlierRunNames.sorted().joinToString(", ")}")
|
||||||
|
}
|
||||||
|
|
||||||
|
val cleanedRuns = originalRuns.filter { it.name !in outlierRunNames }
|
||||||
|
|
||||||
|
println("[boot-stability] cleaned runs=${cleanedRuns.size}")
|
||||||
|
|
||||||
|
if (outlierRunNames.isNotEmpty()) {
|
||||||
|
println("[boot-stability] removed runs=${outlierRunNames.sorted().joinToString(", ")}")
|
||||||
|
}
|
||||||
|
|
||||||
|
val originalStats = requiredStages.associateWith { stage -> statsForStage(originalRuns, stage) }
|
||||||
|
val cleanedStats = requiredStages.associateWith { stage -> statsForStage(cleanedRuns, stage) }
|
||||||
|
|
||||||
|
println("[boot-stability] original stats summary:")
|
||||||
|
for (stage in requiredStages) {
|
||||||
|
val n = originalRuns.mapNotNull { it.stages[stage] }.size
|
||||||
|
val st = originalStats.getValue(stage)
|
||||||
|
if (n == 0) {
|
||||||
|
println("[boot-stability] - $stage: n=0")
|
||||||
|
} else {
|
||||||
|
println("[boot-stability] - $stage: n=$n mean=${format3(st.mean)} std=${format3(st.std)} jitter=${format3(st.jitter)}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println("[boot-stability] cleaned stats summary:")
|
||||||
|
for (stage in requiredStages) {
|
||||||
|
val n = cleanedRuns.mapNotNull { it.stages[stage] }.size
|
||||||
|
val st = cleanedStats.getValue(stage)
|
||||||
|
if (n == 0) {
|
||||||
|
println("[boot-stability] - $stage: n=0")
|
||||||
|
} else {
|
||||||
|
println("[boot-stability] - $stage: n=$n mean=${format3(st.mean)} std=${format3(st.std)} jitter=${format3(st.jitter)}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val outlierListStr = if (outliers.isEmpty()) {
|
||||||
|
"No outliers detected."
|
||||||
|
} else {
|
||||||
|
outliers.joinToString("\n") { o ->
|
||||||
|
"- Run `${o.run}` in stage **${o.stage}**: Value `${format3(o.value)}s` > Threshold `${format3(o.threshold)}s` (Median was ${format3(o.median)}s)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val outlierImpactSummary = when {
|
||||||
|
outliers.isEmpty() -> "- **Outlier Impact**: No outliers were detected, indicating consistent boot times across all runs."
|
||||||
|
outliers.size == 1 -> {
|
||||||
|
val o = outliers[0]
|
||||||
|
"- **Outlier Impact**: The run `${o.run}` was identified as an outlier. The most significant deviation was in the **${o.stage}** stage, " +
|
||||||
|
"with a time of `${format3(o.value)}s`, which is more than double the median of the stable runs (`${format3(o.median)}s`)."
|
||||||
|
}
|
||||||
|
else -> "- **Outlier Impact**: ${outliers.size} outliers were detected across various runs and stages. " +
|
||||||
|
"Refer to the 'Data Cleaning: Outlier Detection' section above for detailed information on each outlier. " +
|
||||||
|
"These outliers represent significant deviations from the median boot times in their respective stages."
|
||||||
|
}
|
||||||
|
|
||||||
|
val outlierRunForConclusion = outlierRunNames.firstOrNull() ?: "N/A"
|
||||||
|
|
||||||
|
val skippedSection = if (skippedRuns.isEmpty()) {
|
||||||
|
""
|
||||||
|
} else {
|
||||||
|
"""
|
||||||
|
## Skipped Runs
|
||||||
|
|
||||||
|
The following runs were skipped because they were missing required inputs:
|
||||||
|
|
||||||
|
${skippedRuns.joinToString("\n") { "- $it" }}
|
||||||
|
|
||||||
|
""".trimIndent() + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
val report = """
|
||||||
|
# Boot Time Stability Analysis Report (kts)
|
||||||
|
|
||||||
|
This report analyzes the boot time of the device based on ${originalRuns.size} runs, with data cleaning applied to remove outliers.
|
||||||
|
A more robust outlier detection method is used: a data point is considered an outlier if its value is greater than 200% of the **median** for that specific boot stage (`value > 2 * median`).
|
||||||
|
|
||||||
|
## 1. Original Data Summary
|
||||||
|
|
||||||
|
### Boot Times (in seconds)
|
||||||
|
${markdownTable(originalRuns)}
|
||||||
|
|
||||||
|
### Statistical Analysis (Original Data, All Runs)
|
||||||
|
${markdownStatsTable(originalStats)}
|
||||||
|
|
||||||
|
## 2. Data Cleaning: Outlier Detection
|
||||||
|
|
||||||
|
The following outliers were detected and removed based on the median rule:
|
||||||
|
|
||||||
|
$outlierListStr
|
||||||
|
|
||||||
|
## 3. Cleaned Data Summary
|
||||||
|
|
||||||
|
### Statistical Analysis (Cleaned Data, ${cleanedRuns.size} Stable Runs)
|
||||||
|
_This analysis excludes the entire run(s) identified as containing outliers._
|
||||||
|
|
||||||
|
${markdownStatsTable(cleanedStats)}
|
||||||
|
|
||||||
|
## 4. Observations
|
||||||
|
|
||||||
|
$outlierImpactSummary
|
||||||
|
- **Stability (Cleaned Data)**: After removing the outlier run, the data for the remaining ${cleanedRuns.size} runs is more stable. For example, the standard deviation for 'Kernel Start' dropped from `${format3(originalStats.getValue("Kernel Start").std)}s` to `${format3(cleanedStats.getValue("Kernel Start").std)}s`.
|
||||||
|
- **Conclusion**: The cleaned data provides an accurate representation of typical boot performance. The outlier run from `$outlierRunForConclusion` should be investigated separately to understand the cause of its significant delay.
|
||||||
|
|
||||||
|
$skippedSection
|
||||||
|
""".trimIndent() + "\n"
|
||||||
|
|
||||||
|
val outFile = File(inputDir, "boot_time_stability_report.md")
|
||||||
|
outFile.writeText(report)
|
||||||
|
|
||||||
|
println("Report generated successfully at ${outFile.absolutePath}")
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,664 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, Iterable, List, Optional, Tuple
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
import matplotlib
|
||||||
|
|
||||||
|
matplotlib.use("Agg")
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ReportRef:
|
||||||
|
build_type: str
|
||||||
|
run_id: str
|
||||||
|
path: Path
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_event_name(name: str) -> str:
|
||||||
|
s = name.strip().strip("`")
|
||||||
|
if s in ("-", "", "N/A"):
|
||||||
|
return ""
|
||||||
|
s = re.sub(r"\s+", " ", s)
|
||||||
|
s = re.sub(r"\s*\(.*?\)\s*$", "", s).strip()
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_seconds(token: str) -> Optional[float]:
|
||||||
|
t = token.strip().strip("`")
|
||||||
|
if t in ("-", "", "N/A"):
|
||||||
|
return None
|
||||||
|
t = t.replace("s", "").strip()
|
||||||
|
try:
|
||||||
|
return float(t)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_markdown_table_rows(lines: List[str]) -> Iterable[List[str]]:
|
||||||
|
for line in lines:
|
||||||
|
stripped = line.strip("\n")
|
||||||
|
if not stripped.strip().startswith("|"):
|
||||||
|
continue
|
||||||
|
if "---" in stripped:
|
||||||
|
continue
|
||||||
|
parts = [p.strip() for p in stripped.split("|")]
|
||||||
|
parts = [p for p in parts if p != ""]
|
||||||
|
if not parts:
|
||||||
|
continue
|
||||||
|
yield parts
|
||||||
|
|
||||||
|
|
||||||
|
def parse_merged_boot_report(path: Path) -> List[Tuple[str, float, int]]:
|
||||||
|
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||||
|
lines = text.splitlines()
|
||||||
|
|
||||||
|
section_idx = None
|
||||||
|
for i, line in enumerate(lines):
|
||||||
|
if line.strip() == "## Merged Timeline Summary":
|
||||||
|
section_idx = i
|
||||||
|
break
|
||||||
|
|
||||||
|
if section_idx is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
table_start = None
|
||||||
|
for i in range(section_idx + 1, len(lines)):
|
||||||
|
if lines[i].lstrip().startswith("|") and "TimeStamp" in lines[i] and "Delta(A)" in lines[i]:
|
||||||
|
table_start = i
|
||||||
|
break
|
||||||
|
|
||||||
|
if table_start is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
table_lines: List[str] = []
|
||||||
|
for i in range(table_start, len(lines)):
|
||||||
|
line = lines[i]
|
||||||
|
if i > table_start and not line.strip():
|
||||||
|
break
|
||||||
|
if line.lstrip().startswith("|"):
|
||||||
|
table_lines.append(line)
|
||||||
|
|
||||||
|
if not table_lines:
|
||||||
|
return []
|
||||||
|
|
||||||
|
header = [h.strip() for h in table_lines[0].split("|")]
|
||||||
|
header = [h for h in header if h]
|
||||||
|
cols: Dict[str, int] = {name: idx for idx, name in enumerate(header)}
|
||||||
|
|
||||||
|
def idx(col: str) -> Optional[int]:
|
||||||
|
return cols.get(col)
|
||||||
|
|
||||||
|
i_log = idx("Log Event")
|
||||||
|
i_vis = idx("Visual Event")
|
||||||
|
i_delta_a = idx("Delta(A)")
|
||||||
|
|
||||||
|
if i_delta_a is None or (i_log is None and i_vis is None):
|
||||||
|
return []
|
||||||
|
|
||||||
|
out: List[Tuple[str, float, int]] = []
|
||||||
|
|
||||||
|
for row_idx, parts in enumerate(_iter_markdown_table_rows(table_lines[1:])):
|
||||||
|
if len(parts) <= i_delta_a:
|
||||||
|
continue
|
||||||
|
|
||||||
|
log_event = parts[i_log] if i_log is not None and len(parts) > i_log else ""
|
||||||
|
vis_event = parts[i_vis] if i_vis is not None and len(parts) > i_vis else ""
|
||||||
|
|
||||||
|
vis_clean = _clean_event_name(vis_event)
|
||||||
|
log_clean = _clean_event_name(log_event)
|
||||||
|
event = vis_clean if vis_clean else log_clean
|
||||||
|
if not event:
|
||||||
|
continue
|
||||||
|
|
||||||
|
t = _parse_seconds(parts[i_delta_a])
|
||||||
|
if t is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
out.append((event, t, row_idx))
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def find_reports(root: Path, build_type: str) -> List[ReportRef]:
|
||||||
|
out: List[ReportRef] = []
|
||||||
|
if not root.exists() or not root.is_dir():
|
||||||
|
return out
|
||||||
|
|
||||||
|
# Only look into first-level subdirectories
|
||||||
|
for p in root.iterdir():
|
||||||
|
if p.is_dir():
|
||||||
|
report_file = p / "merged_boot_report.md"
|
||||||
|
if report_file.exists():
|
||||||
|
run_id = p.name
|
||||||
|
out.append(ReportRef(build_type=build_type, run_id=run_id, path=report_file))
|
||||||
|
|
||||||
|
out.sort(key=lambda r: (r.run_id, str(r.path)))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def build_dataframe(reports: List[ReportRef]) -> pd.DataFrame:
|
||||||
|
rows: List[Dict[str, object]] = []
|
||||||
|
|
||||||
|
for ref in reports:
|
||||||
|
try:
|
||||||
|
events = parse_merged_boot_report(ref.path)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[warn] failed to parse {ref.path}: {e}", file=sys.stderr)
|
||||||
|
continue
|
||||||
|
|
||||||
|
for event, t, raw_idx in events:
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"build_type": ref.build_type,
|
||||||
|
"run_id": ref.run_id,
|
||||||
|
"event": event,
|
||||||
|
"time": float(t),
|
||||||
|
"raw_idx": raw_idx,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
df = pd.DataFrame(rows)
|
||||||
|
if df.empty:
|
||||||
|
return df
|
||||||
|
|
||||||
|
df = df.dropna(subset=["build_type", "run_id", "event", "time"])
|
||||||
|
df["event"] = df["event"].astype(str)
|
||||||
|
df["event_key"] = df["event"].str.lower().str.strip()
|
||||||
|
|
||||||
|
df = (
|
||||||
|
df.sort_values(["build_type", "run_id", "event_key", "time"], ascending=True)
|
||||||
|
.groupby(["build_type", "run_id", "event_key"], as_index=False)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def safe_to_markdown(df: pd.DataFrame) -> str:
|
||||||
|
if df.empty:
|
||||||
|
return "(no data)"
|
||||||
|
try:
|
||||||
|
return df.to_markdown(index=False)
|
||||||
|
except Exception:
|
||||||
|
cols = list(df.columns)
|
||||||
|
rows = df.values.tolist()
|
||||||
|
header = "| " + " | ".join(str(c) for c in cols) + " |\n"
|
||||||
|
sep = "|" + "|".join(["---"] * len(cols)) + "|\n"
|
||||||
|
body = "".join("| " + " | ".join(str(v) for v in r) + " |\n" for r in rows)
|
||||||
|
return header + sep + body
|
||||||
|
|
||||||
|
|
||||||
|
def _format_float(v: object, ndigits: int = 3) -> str:
|
||||||
|
try:
|
||||||
|
if pd.isna(v):
|
||||||
|
return "N/A"
|
||||||
|
return f"{float(v):.{ndigits}f}"
|
||||||
|
except Exception:
|
||||||
|
return "N/A"
|
||||||
|
|
||||||
|
|
||||||
|
def _top_changes(df: pd.DataFrame, abs_col: str, label_col: str, n: int = 5) -> Tuple[pd.DataFrame, pd.DataFrame]:
|
||||||
|
if df.empty or abs_col not in df.columns:
|
||||||
|
return pd.DataFrame(), pd.DataFrame()
|
||||||
|
|
||||||
|
tmp = df.copy()
|
||||||
|
tmp[abs_col] = pd.to_numeric(tmp[abs_col], errors="coerce")
|
||||||
|
tmp = tmp.dropna(subset=[abs_col])
|
||||||
|
if tmp.empty:
|
||||||
|
return pd.DataFrame(), pd.DataFrame()
|
||||||
|
|
||||||
|
improved = tmp[tmp[abs_col] > 0].sort_values(abs_col, ascending=False).head(n)
|
||||||
|
regressed = tmp[tmp[abs_col] < 0].sort_values(abs_col, ascending=True).head(n)
|
||||||
|
|
||||||
|
cols = [c for c in [label_col, abs_col] if c in tmp.columns]
|
||||||
|
return improved[cols], regressed[cols]
|
||||||
|
|
||||||
|
|
||||||
|
def generate_markdown_report(
|
||||||
|
*,
|
||||||
|
out_path: Path,
|
||||||
|
a_name: str,
|
||||||
|
b_name: str,
|
||||||
|
a_dir: Path,
|
||||||
|
b_dir: Path,
|
||||||
|
a_reports: List[ReportRef],
|
||||||
|
b_reports: List[ReportRef],
|
||||||
|
df: pd.DataFrame,
|
||||||
|
all_events_avg: pd.DataFrame,
|
||||||
|
milestones_df: pd.DataFrame,
|
||||||
|
phases_df: pd.DataFrame,
|
||||||
|
milestones_png: Path,
|
||||||
|
phases_png: Path,
|
||||||
|
) -> None:
|
||||||
|
total_runs = df[["build_type", "run_id"]].drop_duplicates().shape[0] if not df.empty else 0
|
||||||
|
runs_by_build = (
|
||||||
|
df[["build_type", "run_id"]].drop_duplicates().groupby("build_type").size().to_dict() if not df.empty else {}
|
||||||
|
)
|
||||||
|
events_by_build = df.groupby("build_type").size().to_dict() if not df.empty else {}
|
||||||
|
|
||||||
|
missing_events = pd.DataFrame()
|
||||||
|
if not df.empty:
|
||||||
|
cover = (
|
||||||
|
df.groupby(["build_type", "event_key"], as_index=False)
|
||||||
|
.size()
|
||||||
|
.rename(columns={"size": "count"})
|
||||||
|
)
|
||||||
|
cover = cover.pivot(index="event_key", columns="build_type", values="count").fillna(0).astype(int)
|
||||||
|
cover = cover.reset_index()
|
||||||
|
cover = cover.rename(columns={"event_key": "event_id"})
|
||||||
|
missing_events = cover[(cover.get(a_name, 0) == 0) | (cover.get(b_name, 0) == 0)].copy()
|
||||||
|
|
||||||
|
top_ms_improve, top_ms_regress = _top_changes(milestones_df, "abs_improvement", "milestone", n=5)
|
||||||
|
top_ph_improve, top_ph_regress = _top_changes(phases_df, "abs_improvement", "phase", n=5)
|
||||||
|
|
||||||
|
with out_path.open("w", encoding="utf-8") as f:
|
||||||
|
f.write(f"# Boot Time A/B Comparison Report\n\n")
|
||||||
|
f.write(f"This report compares boot-time events between **{a_name}** (A) and **{b_name}** (B).\n\n")
|
||||||
|
|
||||||
|
f.write("## 1. Inputs\n\n")
|
||||||
|
f.write(f"- **A**: `{a_name}` @ `{a_dir}`\n")
|
||||||
|
f.write(f"- **B**: `{b_name}` @ `{b_dir}`\n")
|
||||||
|
f.write(f"- **Found reports**: A={len(a_reports)}, B={len(b_reports)}\n")
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
f.write("## 2. Data Coverage\n\n")
|
||||||
|
f.write(f"- **Parsed runs**: {total_runs}\n")
|
||||||
|
f.write(f"- **Runs by build**: A={runs_by_build.get(a_name, 0)}, B={runs_by_build.get(b_name, 0)}\n")
|
||||||
|
f.write(f"- **Parsed event rows**: A={events_by_build.get(a_name, 0)}, B={events_by_build.get(b_name, 0)}\n\n")
|
||||||
|
if not missing_events.empty:
|
||||||
|
f.write("### Events missing in either build (coverage=0)\n\n")
|
||||||
|
f.write(safe_to_markdown(missing_events))
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
f.write("## 3. Average Time for Each Event (All Events)\n\n")
|
||||||
|
f.write(safe_to_markdown(all_events_avg))
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
f.write("## 4. Key Milestones\n\n")
|
||||||
|
f.write(safe_to_markdown(milestones_df))
|
||||||
|
f.write("\n")
|
||||||
|
if milestones_png.exists():
|
||||||
|
f.write("### Milestones Chart\n\n")
|
||||||
|
f.write(f"\n\n")
|
||||||
|
|
||||||
|
f.write("## 5. Boot Phases\n\n")
|
||||||
|
f.write(safe_to_markdown(phases_df))
|
||||||
|
f.write("\n")
|
||||||
|
if phases_png.exists():
|
||||||
|
f.write("### Phases Chart\n\n")
|
||||||
|
f.write(f"\n\n")
|
||||||
|
|
||||||
|
f.write("## 6. Observations\n\n")
|
||||||
|
f.write(f"- **Improvement definition**: `abs_improvement = avg_{b_name} - avg_{a_name}`. Positive means A is faster than B.\n")
|
||||||
|
|
||||||
|
if not top_ms_improve.empty:
|
||||||
|
f.write("\n### Top milestone improvements (A faster)\n\n")
|
||||||
|
f.write(safe_to_markdown(top_ms_improve))
|
||||||
|
f.write("\n")
|
||||||
|
if not top_ms_regress.empty:
|
||||||
|
f.write("\n### Top milestone regressions (A slower)\n\n")
|
||||||
|
f.write(safe_to_markdown(top_ms_regress))
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
if not top_ph_improve.empty:
|
||||||
|
f.write("\n### Top phase improvements (A faster)\n\n")
|
||||||
|
f.write(safe_to_markdown(top_ph_improve))
|
||||||
|
f.write("\n")
|
||||||
|
if not top_ph_regress.empty:
|
||||||
|
f.write("\n### Top phase regressions (A slower)\n\n")
|
||||||
|
f.write(safe_to_markdown(top_ph_regress))
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_time_for_event(run_df: pd.DataFrame, event_key: str) -> Optional[float]:
|
||||||
|
row = run_df.loc[run_df["event_key"] == event_key]
|
||||||
|
if row.empty:
|
||||||
|
return None
|
||||||
|
return float(row.iloc[0]["time"])
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_event_keys(spec: object) -> List[str]:
|
||||||
|
if spec is None:
|
||||||
|
return []
|
||||||
|
if isinstance(spec, str):
|
||||||
|
return [spec]
|
||||||
|
if isinstance(spec, (list, tuple)):
|
||||||
|
out: List[str] = []
|
||||||
|
for x in spec:
|
||||||
|
if isinstance(x, str) and x.strip():
|
||||||
|
out.append(x)
|
||||||
|
return out
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_time_for_any_event(run_df: pd.DataFrame, event_keys: object) -> Optional[float]:
|
||||||
|
for k in _normalize_event_keys(event_keys):
|
||||||
|
t = _pick_time_for_event(run_df, k)
|
||||||
|
if t is not None:
|
||||||
|
return t
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def compute_phase_durations(df: pd.DataFrame, phases: Dict[str, Tuple[object, Optional[object]]]) -> pd.DataFrame:
|
||||||
|
rows: List[Dict[str, object]] = []
|
||||||
|
|
||||||
|
for (build_type, run_id), run_df in df.groupby(["build_type", "run_id"]):
|
||||||
|
for phase_name, (end_keys, start_keys) in phases.items():
|
||||||
|
end_t = _pick_time_for_any_event(run_df, end_keys)
|
||||||
|
start_t = 0.0 if start_keys is None else _pick_time_for_any_event(run_df, start_keys)
|
||||||
|
if end_t is None or start_t is None:
|
||||||
|
continue
|
||||||
|
duration = end_t - start_t
|
||||||
|
if duration < 0:
|
||||||
|
continue
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"build_type": build_type,
|
||||||
|
"run_id": run_id,
|
||||||
|
"phase": phase_name,
|
||||||
|
"duration": duration,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
out = pd.DataFrame(rows)
|
||||||
|
if out.empty:
|
||||||
|
return out
|
||||||
|
out["phase"] = out["phase"].astype(str)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def milestone_summary(df: pd.DataFrame, milestones: List[str], build_types: List[str]) -> pd.DataFrame:
|
||||||
|
if df.empty:
|
||||||
|
return df
|
||||||
|
|
||||||
|
wanted = [m.lower().strip() for m in milestones]
|
||||||
|
sub = df[df["event_key"].isin(wanted)].copy()
|
||||||
|
if sub.empty:
|
||||||
|
return sub
|
||||||
|
|
||||||
|
avg = (
|
||||||
|
sub.groupby(["build_type", "event_key"], as_index=False)
|
||||||
|
.agg({"time": "mean", "raw_idx": "mean"})
|
||||||
|
.rename(columns={"event_key": "event"})
|
||||||
|
)
|
||||||
|
|
||||||
|
avg["event"] = avg["event"].astype(str)
|
||||||
|
avg["time"] = avg["time"].astype(float)
|
||||||
|
|
||||||
|
pivot = avg.pivot(index="event", columns="build_type", values="time")
|
||||||
|
|
||||||
|
a, b = build_types[0], build_types[1]
|
||||||
|
if a not in pivot.columns:
|
||||||
|
pivot[a] = pd.NA
|
||||||
|
if b not in pivot.columns:
|
||||||
|
pivot[b] = pd.NA
|
||||||
|
|
||||||
|
pivot = pivot[[a, b]]
|
||||||
|
pivot = pivot.reset_index()
|
||||||
|
pivot = pivot.rename(columns={"event": "milestone", a: f"avg_{a}", b: f"avg_{b}"})
|
||||||
|
|
||||||
|
# sort by average raw_idx to keep original chronological order
|
||||||
|
order_map = avg.groupby("event")["raw_idx"].mean().to_dict()
|
||||||
|
pivot["sort_key"] = pivot["milestone"].str.lower().str.strip().map(order_map)
|
||||||
|
# If some milestones are not in order_map, they will be NaN in sort_key
|
||||||
|
# We can fill them with a large value to put them at the end, or just sort
|
||||||
|
pivot = pivot.sort_values("sort_key").drop(columns=["sort_key"])
|
||||||
|
|
||||||
|
pivot["abs_improvement"] = pivot[f"avg_{b}"] - pivot[f"avg_{a}"]
|
||||||
|
denom = pd.to_numeric(pivot[f"avg_{b}"], errors="coerce")
|
||||||
|
pivot["pct_improvement"] = (pivot["abs_improvement"] / denom) * 100.0
|
||||||
|
pivot.loc[denom.isna() | (denom <= 0), "pct_improvement"] = pd.NA
|
||||||
|
|
||||||
|
return pivot
|
||||||
|
|
||||||
|
|
||||||
|
def event_average_summary(df: pd.DataFrame, build_types: List[str]) -> pd.DataFrame:
|
||||||
|
if df.empty:
|
||||||
|
return df
|
||||||
|
|
||||||
|
a, b = build_types[0], build_types[1]
|
||||||
|
|
||||||
|
def _mode(series: pd.Series) -> str:
|
||||||
|
s = series.dropna().astype(str)
|
||||||
|
if s.empty:
|
||||||
|
return ""
|
||||||
|
return s.value_counts().index[0]
|
||||||
|
|
||||||
|
display = df.groupby("event_key")["event"].apply(_mode).reset_index(name="event")
|
||||||
|
avg = df.groupby(["build_type", "event_key"], as_index=False).agg({"time": "mean", "raw_idx": "mean"})
|
||||||
|
|
||||||
|
pivot = avg.pivot(index="event_key", columns="build_type", values="time")
|
||||||
|
if a not in pivot.columns:
|
||||||
|
pivot[a] = pd.NA
|
||||||
|
if b not in pivot.columns:
|
||||||
|
pivot[b] = pd.NA
|
||||||
|
|
||||||
|
pivot = pivot[[a, b]].reset_index()
|
||||||
|
pivot = pivot.rename(columns={a: f"avg_{a}", b: f"avg_{b}"})
|
||||||
|
pivot = pivot.merge(display, on="event_key", how="left")
|
||||||
|
|
||||||
|
# sort by average raw_idx to keep original chronological order
|
||||||
|
order_map = avg.groupby("event_key")["raw_idx"].mean().sort_values().index.tolist()
|
||||||
|
pivot["event_key"] = pd.Categorical(pivot["event_key"], categories=order_map, ordered=True)
|
||||||
|
pivot = pivot.sort_values("event_key")
|
||||||
|
pivot = pivot[["event", "event_key", f"avg_{a}", f"avg_{b}"]]
|
||||||
|
pivot["event_key"] = pivot["event_key"].astype(str) # Convert back to string to avoid categorical issues later
|
||||||
|
pivot = pivot.rename(columns={"event_key": "event_id"})
|
||||||
|
return pivot
|
||||||
|
|
||||||
|
|
||||||
|
def phase_summary(phase_df: pd.DataFrame, build_types: List[str], phases: Dict[str, object]) -> pd.DataFrame:
|
||||||
|
if phase_df.empty:
|
||||||
|
return phase_df
|
||||||
|
|
||||||
|
avg = (
|
||||||
|
phase_df.groupby(["build_type", "phase"], as_index=False)["duration"]
|
||||||
|
.mean()
|
||||||
|
.rename(columns={"duration": "avg_duration"})
|
||||||
|
)
|
||||||
|
|
||||||
|
pivot = avg.pivot(index="phase", columns="build_type", values="avg_duration")
|
||||||
|
a, b = build_types[0], build_types[1]
|
||||||
|
if a not in pivot.columns:
|
||||||
|
pivot[a] = pd.NA
|
||||||
|
if b not in pivot.columns:
|
||||||
|
pivot[b] = pd.NA
|
||||||
|
pivot = pivot[[a, b]].reset_index()
|
||||||
|
pivot = pivot.rename(columns={a: f"avg_{a}", b: f"avg_{b}"})
|
||||||
|
|
||||||
|
# since phases are defined in a dictionary, they follow the defined sequence.
|
||||||
|
# We should respect the dictionary order for the final table and stacked chart.
|
||||||
|
phase_order = list(phases.keys())
|
||||||
|
pivot["phase"] = pd.Categorical(pivot["phase"], categories=phase_order, ordered=True)
|
||||||
|
pivot = pivot.sort_values("phase")
|
||||||
|
pivot["phase"] = pivot["phase"].astype(str) # Convert back to string to avoid categorical issues later
|
||||||
|
|
||||||
|
pivot["abs_improvement"] = pivot[f"avg_{b}"] - pivot[f"avg_{a}"]
|
||||||
|
denom = pd.to_numeric(pivot[f"avg_{b}"], errors="coerce")
|
||||||
|
pivot["pct_improvement"] = (pivot["abs_improvement"] / denom) * 100.0
|
||||||
|
pivot.loc[denom.isna() | (denom <= 0), "pct_improvement"] = pd.NA
|
||||||
|
return pivot
|
||||||
|
|
||||||
|
|
||||||
|
def plot_milestones(milestone_df: pd.DataFrame, a: str, b: str, out_path: Path) -> None:
|
||||||
|
if milestone_df.empty:
|
||||||
|
return
|
||||||
|
|
||||||
|
labels = milestone_df["milestone"].astype(str).tolist()
|
||||||
|
y = list(range(len(labels)))
|
||||||
|
|
||||||
|
a_vals = pd.to_numeric(milestone_df[f"avg_{a}"], errors="coerce").fillna(0.0)
|
||||||
|
b_vals = pd.to_numeric(milestone_df[f"avg_{b}"], errors="coerce").fillna(0.0)
|
||||||
|
|
||||||
|
height = 0.35
|
||||||
|
|
||||||
|
fig, ax = plt.subplots(figsize=(10, max(3.5, 0.5 * len(labels))))
|
||||||
|
ax.barh([yy - height / 2 for yy in y], a_vals, height=height, label=a)
|
||||||
|
ax.barh([yy + height / 2 for yy in y], b_vals, height=height, label=b)
|
||||||
|
|
||||||
|
ax.set_yticks(y)
|
||||||
|
ax.set_yticklabels(labels)
|
||||||
|
ax.invert_yaxis()
|
||||||
|
ax.set_xlabel("Average time (s)")
|
||||||
|
ax.set_title(f"Boot milestones comparison: {a} vs {b}")
|
||||||
|
ax.legend()
|
||||||
|
fig.tight_layout()
|
||||||
|
fig.savefig(out_path, dpi=150)
|
||||||
|
plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def plot_phases(phase_df: pd.DataFrame, a: str, b: str, out_path: Path) -> None:
|
||||||
|
if phase_df.empty:
|
||||||
|
return
|
||||||
|
|
||||||
|
phases = phase_df["phase"].astype(str).tolist()
|
||||||
|
|
||||||
|
avg_a = pd.to_numeric(phase_df[f"avg_{a}"], errors="coerce").fillna(0.0).tolist()
|
||||||
|
avg_b = pd.to_numeric(phase_df[f"avg_{b}"], errors="coerce").fillna(0.0).tolist()
|
||||||
|
|
||||||
|
fig, ax = plt.subplots(figsize=(10, 4.5))
|
||||||
|
|
||||||
|
bottoms_a = 0.0
|
||||||
|
bottoms_b = 0.0
|
||||||
|
|
||||||
|
cmap = plt.get_cmap("tab20")
|
||||||
|
for i, ph in enumerate(phases):
|
||||||
|
color = cmap(i % 20)
|
||||||
|
ax.bar([a, b], [avg_a[i], avg_b[i]], bottom=[bottoms_a, bottoms_b], color=color, label=ph)
|
||||||
|
bottoms_a += avg_a[i]
|
||||||
|
bottoms_b += avg_b[i]
|
||||||
|
|
||||||
|
ax.set_ylabel("Average duration (s)")
|
||||||
|
ax.set_title(f"Boot phases (stacked): {a} vs {b}")
|
||||||
|
|
||||||
|
ax.legend(bbox_to_anchor=(1.02, 1), loc="upper left")
|
||||||
|
|
||||||
|
fig.tight_layout()
|
||||||
|
fig.savefig(out_path, dpi=150)
|
||||||
|
plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Optional[List[str]] = None) -> int:
|
||||||
|
p = argparse.ArgumentParser(prog="boot_analysis.py")
|
||||||
|
p.add_argument("--a-name", required=True, help="Name of build A (baseline)")
|
||||||
|
p.add_argument("--a-dir", required=True, help="Directory of build A (will be searched recursively)")
|
||||||
|
p.add_argument("--b-name", required=True, help="Name of build B (comparison)")
|
||||||
|
p.add_argument("--b-dir", required=True, help="Directory of build B (will be searched recursively)")
|
||||||
|
p.add_argument("--out-dir", default=".", help="Output directory for PNG charts")
|
||||||
|
p.add_argument("--report-name", default="boot_ab_compare_report.md", help="Markdown report filename under out-dir")
|
||||||
|
p.add_argument(
|
||||||
|
"--milestone",
|
||||||
|
action="append",
|
||||||
|
dest="milestones",
|
||||||
|
help="Milestone event name (can be repeated). Default: Boot Logo, bootanim start, bootanim end, Launcher start, Launcher loaded",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = p.parse_args(argv)
|
||||||
|
|
||||||
|
a_name = args.a_name
|
||||||
|
b_name = args.b_name
|
||||||
|
a_dir = Path(args.a_dir)
|
||||||
|
b_dir = Path(args.b_dir)
|
||||||
|
out_dir = Path(args.out_dir)
|
||||||
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
report_name = args.report_name
|
||||||
|
|
||||||
|
milestones = args.milestones or [
|
||||||
|
"Boot Logo",
|
||||||
|
"bootanim start",
|
||||||
|
"bootanim end",
|
||||||
|
"Launcher start",
|
||||||
|
"Launcher loaded",
|
||||||
|
]
|
||||||
|
|
||||||
|
phases: Dict[str, Tuple[object, Optional[object]]] = {
|
||||||
|
"Boot Logo": ("boot logo", None),
|
||||||
|
"Animation Start": ("bootanim start", "boot logo"),
|
||||||
|
"Animation Duration": ("bootanim end", "bootanim start"),
|
||||||
|
"Launcher Loading": (["launcher loaded", "launcher start"], "bootanim end"),
|
||||||
|
}
|
||||||
|
|
||||||
|
a_reports = find_reports(a_dir, a_name)
|
||||||
|
b_reports = find_reports(b_dir, b_name)
|
||||||
|
|
||||||
|
print(f"[info] A: name={a_name} dir={a_dir} reports={len(a_reports)}")
|
||||||
|
print(f"[info] B: name={b_name} dir={b_dir} reports={len(b_reports)}")
|
||||||
|
|
||||||
|
df = build_dataframe(a_reports + b_reports)
|
||||||
|
if df.empty:
|
||||||
|
print("[error] no events parsed", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
print("\n## Average time for each event (all events)\n")
|
||||||
|
ev = event_average_summary(df, build_types=[a_name, b_name])
|
||||||
|
ev_display = ev.copy()
|
||||||
|
if not ev_display.empty:
|
||||||
|
for c in [f"avg_{a_name}", f"avg_{b_name}"]:
|
||||||
|
if c in ev_display.columns:
|
||||||
|
ev_display[c] = pd.to_numeric(ev_display[c], errors="coerce").round(3)
|
||||||
|
print(safe_to_markdown(ev_display))
|
||||||
|
|
||||||
|
print("\n## Average milestone timings\n")
|
||||||
|
ms = milestone_summary(df, milestones, build_types=[a_name, b_name])
|
||||||
|
ms_display = ms.copy()
|
||||||
|
if not ms_display.empty:
|
||||||
|
for c in [f"avg_{a_name}", f"avg_{b_name}", "abs_improvement", "pct_improvement"]:
|
||||||
|
if c in ms_display.columns:
|
||||||
|
ms_display[c] = pd.to_numeric(ms_display[c], errors="coerce").round(3)
|
||||||
|
print(safe_to_markdown(ms_display))
|
||||||
|
|
||||||
|
phase_df = compute_phase_durations(df, phases)
|
||||||
|
print("\n## Average phase durations\n")
|
||||||
|
ph = phase_summary(phase_df, build_types=[a_name, b_name], phases=phases)
|
||||||
|
ph_display = ph.copy()
|
||||||
|
if not ph_display.empty:
|
||||||
|
for c in [f"avg_{a_name}", f"avg_{b_name}", "abs_improvement", "pct_improvement"]:
|
||||||
|
if c in ph_display.columns:
|
||||||
|
ph_display[c] = pd.to_numeric(ph_display[c], errors="coerce").round(3)
|
||||||
|
print(safe_to_markdown(ph_display))
|
||||||
|
|
||||||
|
milestones_png = out_dir / "boot_milestones_comparison_v2.png"
|
||||||
|
if not ms.empty:
|
||||||
|
plot_milestones(ms, a_name, b_name, milestones_png)
|
||||||
|
print(f"\n[info] saved: {milestones_png}")
|
||||||
|
|
||||||
|
phases_png = out_dir / "boot_phases_comparison_v2.png"
|
||||||
|
if not ph.empty:
|
||||||
|
plot_phases(ph, a_name, b_name, phases_png)
|
||||||
|
print(f"[info] saved: {phases_png}")
|
||||||
|
|
||||||
|
report_path = out_dir / report_name
|
||||||
|
try:
|
||||||
|
generate_markdown_report(
|
||||||
|
out_path=report_path,
|
||||||
|
a_name=a_name,
|
||||||
|
b_name=b_name,
|
||||||
|
a_dir=a_dir,
|
||||||
|
b_dir=b_dir,
|
||||||
|
a_reports=a_reports,
|
||||||
|
b_reports=b_reports,
|
||||||
|
df=df,
|
||||||
|
all_events_avg=ev_display,
|
||||||
|
milestones_df=ms_display,
|
||||||
|
phases_df=ph_display,
|
||||||
|
milestones_png=milestones_png,
|
||||||
|
phases_png=phases_png,
|
||||||
|
)
|
||||||
|
print(f"[info] saved: {report_path}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[warn] failed to write markdown report: {e}", file=sys.stderr)
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Loading…
Reference in New Issue