diff --git a/lazybox/src/main/kotlin/cfig/lazybox/App.kt b/lazybox/src/main/kotlin/cfig/lazybox/App.kt index a65823e..0400dfc 100644 --- a/lazybox/src/main/kotlin/cfig/lazybox/App.kt +++ b/lazybox/src/main/kotlin/cfig/lazybox/App.kt @@ -2,7 +2,6 @@ package cfig.lazybox import cfig.lazybox.profiler.VideoAnalyzer import cfig.lazybox.staging.AospCompiledb -import cfig.lazybox.staging.DiffCI import cfig.lazybox.staging.Perfetto import cfig.lazybox.staging.RepoWorker import cfig.lazybox.sysinfo.BootChart @@ -41,7 +40,6 @@ fun main(args: Array) { println("\nIncubating usage:") println("compiledb : generate compilation database for AOSP") println("dmainfo : parse /d/dma_buf/bufinfo") - println("diffci : find changelist files from CI server based on date and time ranges") println("repo_lfs : pull LFS files from Git repositories managed by 'repo'") println("repo_unshallow: unshallow Git repositories managed by 'repo'") println("perfetto : generate a Perfetto configuration file") @@ -119,9 +117,6 @@ fun main(args: Array) { log.error("File not found: $dmainfoFile") } } - if (args[0] == "diffci") { - DiffCI().run(args.drop(1).toTypedArray()) - } if (args[0] == "repo_lfs") { RepoWorker().lfsPullRepo(args.drop(1).toTypedArray()) } diff --git a/lazybox/src/main/kotlin/cfig/lazybox/staging/DiffCI.kt b/lazybox/src/main/kotlin/cfig/lazybox/staging/DiffCI.kt deleted file mode 100644 index bdcca8f..0000000 --- a/lazybox/src/main/kotlin/cfig/lazybox/staging/DiffCI.kt +++ /dev/null @@ -1,307 +0,0 @@ -package cfig.lazybox.staging - -import org.slf4j.LoggerFactory -import java.io.File -import java.io.IOException -import java.net.HttpURLConnection -import java.net.MalformedURLException -import java.net.URI -import java.net.URL -import java.net.http.HttpClient -import java.net.http.HttpRequest -import java.net.http.HttpResponse -import java.time.LocalDate -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter -import java.time.Duration -import java.time.format.DateTimeParseException -import kotlin.system.exitProcess - -class DiffCI { - private val FAKE_URL = "FAKE_URL" - private val baseUrl = System.getenv("diffCIbaseUrl") ?: FAKE_URL - private val finder = ChangelistFinder(baseUrl) - private val dateFormat = DateTimeFormatter.ofPattern("yyyyMMdd") - private val dateTimeFormat = DateTimeFormatter.ofPattern("yyyyMMddHHmm") - private val buildIdRegex = ".*/(\\d{12})/changelist".toRegex() - private val log = LoggerFactory.getLogger(DiffCI::class.simpleName) - - fun run(args: Array) { - if (baseUrl == FAKE_URL) { - log.info("Error: Environment variable 'diffCIbaseUrl' is not set.") - log.info("Please set it to the base URL of the changelist directory.") - exitProcess(1) - } - when (args.size) { - 1 -> handleSingleDate(args) - 2 -> handleRange(args) - else -> printUsageAndExit() - } - } - - private fun handleSingleDate(args: Array) { - if (args[0].length != 8) { - log.info("Error: For a single argument, please provide a date in yyyyMMdd format.") - exitProcess(1) - } - val targetDate = parseDate(args[0]) ?: exitProcess(1) - - log.info("=============================================") - log.info("Searching for changelists on: ${args[0]}") - log.info("=============================================") - - val urls = finder.findChangelistUrlsForDate(targetDate) - printResults(urls) - } - - private fun handleRange(args: Array) { - val arg1 = args[0] - val arg2 = args[1] - - if (arg1.length != arg2.length) { - log.info("Error: Start and end arguments must be of the same format (both dates or both date-times).") - exitProcess(1) - } - - when (arg1.length) { - 8 -> handleDateRange(arg1, arg2) - 12 -> handleDateTimeRange(arg1, arg2) - else -> { - log.info("Error: Invalid argument format. Please use yyyyMMdd or yyyyMMddHHmm.") - exitProcess(1) - } - } - } - - private fun handleDateRange(startDateStr: String, endDateStr: String) { - val startDate = parseDate(startDateStr) ?: exitProcess(1) - val endDate = parseDate(endDateStr) ?: exitProcess(1) - - if (startDate.isAfter(endDate)) { - log.info("Error: The start date ($startDateStr) must be before or the same as the end date ($endDateStr).") - exitProcess(1) - } - - log.info("=============================================") - log.info("Searching for changelists from $startDateStr to $endDateStr") - log.info("=============================================") - - val allUrls = mutableListOf() - var currentDate = startDate - while (currentDate <= endDate) { - log.info("----- Processing Date: $currentDate -----") - val dailyUrls = finder.findChangelistUrlsForDate(currentDate) - allUrls.addAll(dailyUrls) - currentDate = currentDate.plusDays(1) - } - printResults(allUrls) - } - - private fun handleDateTimeRange(startDateTimeStr: String, endDateTimeStr: String) { - val startDateTime = parseDateTime(startDateTimeStr) ?: exitProcess(1) - val endDateTime = parseDateTime(endDateTimeStr) ?: exitProcess(1) - - if (startDateTime.isAfter(endDateTime)) { - log.info("Error: The start time ($startDateTimeStr) must be before or the same as the end time ($endDateTimeStr).") - exitProcess(1) - } - - log.info("=============================================") - log.info("Searching for changelists between $startDateTimeStr and $endDateTimeStr") - log.info("=============================================") - - val allUrls = mutableListOf() - var currentDate = startDateTime.toLocalDate() - while (currentDate <= endDateTime.toLocalDate()) { - log.info("----- Processing Date: $currentDate -----") - val dailyUrls = finder.findChangelistUrlsForDate(currentDate) - - val filteredUrls = dailyUrls.filter { url -> - val matchResult = buildIdRegex.find(url) - if (matchResult != null) { - val buildId = matchResult.groupValues[1] - val buildDateTime = parseDateTime(buildId) - buildDateTime != null && !buildDateTime.isBefore(startDateTime) && !buildDateTime.isAfter(endDateTime) - } else { - false - } - } - allUrls.addAll(filteredUrls) - currentDate = currentDate.plusDays(1) - } - printResults(allUrls, isTimeRange = true) - } - - private fun printUsageAndExit() { - log.info("Error: Invalid number of arguments.") - log.info("Usage:") - log.info(" Single Date: --args='diffci '") - log.info(" Date Range: --args='diffci '") - log.info(" Time Range: --args='diffci '") - log.info("Examples:") - log.info(" --args='diffci 20250628'") - log.info(" --args='diffci 20250627 20250628'") - log.info(" --args='diffci 202506281000 202506281430'") - exitProcess(1) - } - - fun fetchChangeList(urlsToFetch: List) { - val OUTPUT_FILENAME = "merged_content.txt" - if (urlsToFetch.isEmpty()) { - return - } - val outputFile = File(OUTPUT_FILENAME) - log.info("Starting to fetch content from ${urlsToFetch.size} URLs ...") - - outputFile.bufferedWriter().use { writer -> - // Iterate over each URL string in the list. - urlsToFetch.forEachIndexed { index, urlString -> - log.info("(${index + 1}/${urlsToFetch.size}) Fetching: $urlString") - try { - // Create a URL object from the string. - val url = URL(urlString) - val connection = (url.openConnection() as HttpURLConnection).also { - it.requestMethod = "GET" - it.connectTimeout = 5000 // 5 seconds - it.readTimeout = 5000 // 5 seconds - } - val responseCode = connection.responseCode - // Write a header for this URL's content in the output file. - writer.write("--- START: Content from $urlString ---") - writer.newLine() - // Check for a successful HTTP response (2xx status codes). - if (responseCode in 200..299) { - connection.inputStream.bufferedReader().use { reader -> - // Read the content line by line and write to the file. - reader.forEachLine { line -> - writer.write(line) - writer.newLine() - } - } - } else { - val errorMessage = "Received non-OK response code: $responseCode" - log.info("Warning: $errorMessage for $urlString") - writer.write("! FAILED: $errorMessage") - writer.newLine() - } - - } catch (e: MalformedURLException) { - val errorMessage = "Invalid URL format" - log.error("Error for '$urlString': $errorMessage. Skipping.") - writer.write("! FAILED: $errorMessage") - writer.newLine() - } catch (e: IOException) { - val errorMessage = "Error reading data (I/O problem): ${e.message}" - log.info("Error for '$urlString': $errorMessage. Skipping.") - writer.write("! FAILED: $errorMessage") - writer.newLine() - } catch (e: Exception) { - val errorMessage = "An unexpected error occurred: ${e.message}" - log.error("Error for '$urlString': $errorMessage. Skipping.") - writer.write("! FAILED: $errorMessage") - writer.newLine() - } finally { - // Write a footer and add extra newlines for separation. - writer.write("--- END: Content from $urlString ---") - writer.newLine() - writer.newLine() - } - } - } - log.info("Process complete. '${outputFile.absolutePath}'.") - } - - private fun printResults(urls: List, isTimeRange: Boolean = false) { - log.info("--- Results ---") - if (urls.isNotEmpty()) { - log.info("Successfully found ${urls.size} changelist files:") - urls.forEach { log.info(it) } - } else { - val rangeType = if (isTimeRange) "time range" else "date(s)" - log.info("No changelist files were found for the specified $rangeType.") - } - log.info("---------------") - fetchChangeList(urls) - } - - private fun parseDate(dateStr: String): LocalDate? { - return try { - LocalDate.parse(dateStr, dateFormat) - } catch (e: DateTimeParseException) { - log.info("Error: Invalid date format for '$dateStr'.") - log.info("Please use the yyyyMMdd format (e.g., 20250628).") - null - } - } - - private fun parseDateTime(dateTimeStr: String): LocalDateTime? { - return try { - LocalDateTime.parse(dateTimeStr, dateTimeFormat) - } catch (e: DateTimeParseException) { - log.info("Error: Invalid date-time format for '$dateTimeStr'.") - log.info("Please use the yyyyMMddHHmm format (e.g., 202506281027).") - null - } - } - - /** - * Inner class to handle the web scraping logic. - */ - private inner class ChangelistFinder(private val baseUrl: String) { - private val client = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(Duration.ofSeconds(10)) - .build() - - fun findChangelistUrlsForDate(date: LocalDate): List { - val yearMonthPattern = DateTimeFormatter.ofPattern("yyyyMM") - val yearMonthDayPattern = DateTimeFormatter.ofPattern("yyyyMMdd") - val yearMonthStr = date.format(yearMonthPattern) - val yearMonthDayStr = date.format(yearMonthDayPattern) - val directoryUrl = "$baseUrl/$yearMonthStr/$yearMonthDayStr/" - - return try { - val htmlContent = fetchUrlContent(directoryUrl) - if (htmlContent != null) { - parseDirectoryHtmlWithRegex(htmlContent, directoryUrl, yearMonthDayStr) - } else { - emptyList() - } - } catch (e: IOException) { - log.info("An error occurred during the network request: ${e.message}") - emptyList() - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - log.info("The network request was interrupted: ${e.message}") - emptyList() - } - } - - private fun fetchUrlContent(url: String): String? { - try { - val request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build() - val response = client.send(request, HttpResponse.BodyHandlers.ofString()) - if (response.statusCode() != 200) { - if (response.statusCode() != 404) { - log.info("Request for $url failed with code: ${response.statusCode()}") - } - return null - } - return response.body() - } catch (e: Exception) { - log.info("Exception while fetching URL '$url': ${e.message}") - return null - } - } - - private fun parseDirectoryHtmlWithRegex(html: String, directoryUrl: String, datePrefix: String): List { - val regex = "href=\"($datePrefix\\d+)/\"".toRegex() - return regex.findAll(html) - .map { matchResult -> - val dirName = matchResult.groupValues[1] - "$directoryUrl$dirName/changelist" - } - .toList() - } - } -}