From 3bf5977e8120c8807a64a02e914a4502c7bc519a Mon Sep 17 00:00:00 2001 From: Dmitry Muhomor Date: Sat, 26 Aug 2023 18:26:49 +0300 Subject: [PATCH] add command for making "BUILD_ID to AOSP tag" map --- config/build-index/build-id-to-tag.yml | 72 +++++++++++++++ src/commands/update-aosp-tag-index.ts | 121 +++++++++++++++++++++++++ src/config/paths.ts | 2 + src/util/git.ts | 39 ++++++++ 4 files changed, 234 insertions(+) create mode 100644 config/build-index/build-id-to-tag.yml create mode 100644 src/commands/update-aosp-tag-index.ts create mode 100644 src/util/git.ts diff --git a/config/build-index/build-id-to-tag.yml b/config/build-index/build-id-to-tag.yml new file mode 100644 index 0000000..eecab91 --- /dev/null +++ b/config/build-index/build-id-to-tag.yml @@ -0,0 +1,72 @@ +TP1A.220624.014: android-13.0.0_r1 +TP1A.220624.021: android-13.0.0_r2 +TP1A.220624.021.A1: android-13.0.0_r3 +TP1A.220905.004: android-13.0.0_r4 +TP1A.220905.004.A1: android-13.0.0_r5 +TP1A.220905.004.A2: android-13.0.0_r6 +TP1A.221005.002: android-13.0.0_r7 +TP1A.221005.003: android-13.0.0_r8 +TD1A.220804.009.A2: android-13.0.0_r9 +TD1A.220804.009.A5: android-13.0.0_r10 +TD1A.220804.031: android-13.0.0_r11 +TP1A.221105.002: android-13.0.0_r12 +TD1A.221105.001: android-13.0.0_r13 +TD1A.221105.003: android-13.0.0_r14 +TD1A.221105.001.A1: android-13.0.0_r15 +TQ1A.221205.011: android-13.0.0_r16 +TQ1A.221205.011.B1: android-13.0.0_r17 +TQ1A.221205.012: android-13.0.0_r18 +TQ1A.230105.001: android-13.0.0_r19 +TQ1A.230105.001.A2: android-13.0.0_r20 +TQ1A.230105.001.A3: android-13.0.0_r21 +TQ1A.230105.001.B1: android-13.0.0_r22 +TQ1A.230105.002: android-13.0.0_r23 +TQ1A.230105.002.A1: android-13.0.0_r24 +TQ1A.230205.001.A2: android-13.0.0_r27 +TQ1A.230205.001.B2: android-13.0.0_r28 +TQ1A.230205.001.D2: android-13.0.0_r29 +TQ1A.230205.002: android-13.0.0_r30 +TP1A.221005.002.B2: android-13.0.0_r31 +TQ2A.230305.008: android-13.0.0_r32 +TQ2A.230305.008.A1: android-13.0.0_r33 +TQ2A.230305.008.A3: android-13.0.0_r34 +TQ2A.230305.008.C1: android-13.0.0_r35 +TQ2A.230305.008.E1: android-13.0.0_r36 +TQ2A.230305.008.F1: android-13.0.0_r37 +TQ2A.230405.003: android-13.0.0_r38 +TQ2A.230405.003.A2: android-13.0.0_r39 +TQ2A.230405.003.B2: android-13.0.0_r40 +TQ2A.230405.003.E1: android-13.0.0_r41 +TQ2A.230405.003.G1: android-13.0.0_r42 +TQ2A.230505.002: android-13.0.0_r43 +TQ2A.230505.002.A1: android-13.0.0_r44 +TQ2A.230505.002.G1: android-13.0.0_r45 +TD4A.221205.042: android-13.0.0_r46 +TD4A.221205.042.A1: android-13.0.0_r47 +TD4A.221205.042.B1: android-13.0.0_r48 +TQ2B.230505.005.A1: android-13.0.0_r49 +TQ3A.230605.010: android-13.0.0_r50 +TQ3A.230605.011: android-13.0.0_r51 +TQ3A.230605.012: android-13.0.0_r52 +TQ3A.230605.010.A1: android-13.0.0_r53 +TQ3A.230605.012.A1: android-13.0.0_r54 +TD2A.230203.028: android-13.0.0_r55 +TQ3A.230605.009.A1: android-13.0.0_r56 +TD3A.230203.070.A1: android-13.0.0_r57 +TQ3C.230605.010.C1: android-13.0.0_r58 +TQ3C.230605.010.C2: android-13.0.0_r59 +TQ3C.230605.010.D1: android-13.0.0_r60 +TQ3A.230705.001: android-13.0.0_r61 +TQ3A.230705.001.A1: android-13.0.0_r62 +TQ3A.230705.001.B4: android-13.0.0_r63 +TQ3C.230705.001.B1: android-13.0.0_r64 +TQ3C.230705.001.C1: android-13.0.0_r65 +TQ3C.230705.001.C2: android-13.0.0_r66 +TQ3A.230805.001: android-13.0.0_r67 +TQ3A.230805.001.A1: android-13.0.0_r68 +TQ3A.230805.001.A2: android-13.0.0_r69 +TQ3A.230805.001.A3: android-13.0.0_r70 +TQ3A.230805.001.B1: android-13.0.0_r71 +TQ3C.230805.001.A3: android-13.0.0_r72 +TQ3C.230805.001.A4: android-13.0.0_r73 +TQ3C.230805.001.B2: android-13.0.0_r74 diff --git a/src/commands/update-aosp-tag-index.ts b/src/commands/update-aosp-tag-index.ts new file mode 100644 index 0000000..3bfc07c --- /dev/null +++ b/src/commands/update-aosp-tag-index.ts @@ -0,0 +1,121 @@ +import { Command, flags } from '@oclif/command' +import assert from 'assert' +import { writeFileSync } from 'fs' +import YAML from 'yaml' + +import { ADEVTOOL_DIR, BUILD_ID_TO_TAG_FILE } from '../config/paths' +import { showGitDiff } from '../util/cli' +import { readFile } from '../util/fs' +import { GitLsRemote } from '../util/git' +import { yamlStringifyNoFold } from '../util/yaml' + +type BuildIdToTag = Map + +export async function loadBuildIdToTagMap(): Promise { + try { + let file = await readFile(BUILD_ID_TO_TAG_FILE) + return YAML.parse(file, { mapAsMap: true }) as BuildIdToTag + } catch { + return null + } +} + +export default class UpdateAospTagIndex extends Command { + static description = `update buildId-to-tag map in ${BUILD_ID_TO_TAG_FILE}` + + static flags = { + dryRun: flags.boolean({ + char: 'n', + description: 'skip file update', + }), + } + + async run() { + let { flags } = this.parse(UpdateAospTagIndex) + + // let bases = ['android-vts', 'android-security', 'android', 'android-platform', 'android-cts'] + let bases = ['android'] + // let versions = ['11.0.0', '12.0.0', '12.1.0', '13.0.0'] + let versions = ['13.0.0', '14.0.0'] + + let prefixes: string[] = [] + bases.forEach(base => + versions.forEach(version => { + if (base === 'android-cts' || base === 'android-vts') { + version = version.slice(0, -2) + } + prefixes.push(`${base}-${version}_r`) + }), + ) + + let map = (await loadBuildIdToTagMap()) ?? new Map() + let lsRemote = await GitLsRemote.get('https://github.com/aosp-mirror/platform_build') + + let knownTags = new Set(map.values()) + let requests = new Map>() + + for (let tagName of lsRemote.tags.keys()) { + if (knownTags.has(tagName)) { + continue + } + + if (prefixes.find(p => tagName.startsWith(p)) === undefined) { + continue + } + + let url = `https://raw.githubusercontent.com/aosp-mirror/platform_build/${tagName}/core/build_id.mk` + requests.set(tagName, fetch(url)) + } + + let numNewTags = 0 + + for (let [tagName, responsePromise] of requests.entries()) { + let resp = await responsePromise + assert(resp.ok, tagName) + + let build_id_mk = await resp.text() + let buildIdPrefix = 'BUILD_ID=' + let line = build_id_mk.split('\n').find(s => s.startsWith(buildIdPrefix)) + assert(line !== undefined, 'missing BUILD_ID line in:\n' + build_id_mk) + let build_id = line.substring(buildIdPrefix.length).trim() + + this.log(tagName + ': ' + build_id) + map.set(build_id, tagName) + ++numNewTags + } + + if (!flags.dryRun && numNewTags > 0) { + let entries = Array.from(map.entries()) + .sort(([_k1, tag1], [_k2, tag2]) => compareTags(tag1, tag2)) + + writeFileSync(BUILD_ID_TO_TAG_FILE, yamlStringifyNoFold(new Map(entries))) + } + + if (numNewTags === 0) { + this.log('No new tags') + process.exit(1) + } + + showGitDiff(ADEVTOOL_DIR, BUILD_ID_TO_TAG_FILE) + } +} + +function getTagRevision(tag: string) { + let idx = tag.lastIndexOf('_r') + assert(idx > 0, tag) + return parseInt(tag.substring(idx + 2)) +} + +function getTagBase(tag: string) { + let idx = tag.lastIndexOf('_r') + assert(idx > 0, tag) + return tag.slice(0, idx) +} + +function compareTags(a: string, b: string) { + let res = getTagBase(a).localeCompare(getTagBase(b)) + if (res == 0) { + res = getTagRevision(a) - getTagRevision(b) + } + return res +} diff --git a/src/config/paths.ts b/src/config/paths.ts index 811f9ba..608b888 100644 --- a/src/config/paths.ts +++ b/src/config/paths.ts @@ -21,3 +21,5 @@ export const BUILD_INDEX_FILE = path.join(BUILD_INDEX_DIR, 'build-index.yml') export const MAIN_BUILD_INDEX_PART = path.join(BUILD_INDEX_DIR, 'build-index-main.yml') export const IMAGE_DOWNLOAD_DIR = process.env['ADEVTOOL_IMG_DOWNLOAD_DIR'] ?? path.join(ADEVTOOL_DIR, 'dl') + +export const BUILD_ID_TO_TAG_FILE = path.join(BUILD_INDEX_DIR, 'build-id-to-tag.yml') diff --git a/src/util/git.ts b/src/util/git.ts new file mode 100644 index 0000000..d2e9cde --- /dev/null +++ b/src/util/git.ts @@ -0,0 +1,39 @@ +import assert from 'assert' +import { spawnAsync } from './process' + +export class GitLsRemote { + tags = new Map() + branches = new Map() + + static async get(repoUrl: string) { + let lsRemote = await spawnAsync('git', ['ls-remote', '--tags', '--heads', repoUrl]) + + let result = new GitLsRemote() + + const tagPrefix = 'refs/tags/' + const tagSuffix = '^{}' + const branchPrefix = 'refs/heads/' + + for (let line of (await lsRemote).split('\n')) { + if (line.length === 0) { + continue + } + let split = line.split('\t') + assert(split.length == 2, line) + let commit = split[0] + let desc = split[1] + + if (desc.startsWith(tagPrefix)) { + if (!desc.endsWith(tagSuffix)) { + continue + } + result.tags.set(desc.slice(tagPrefix.length, - tagSuffix.length), commit) + } else { + assert(desc.startsWith(branchPrefix), line) + result.branches.set(desc, commit) + } + } + + return result + } +}