add command for making "BUILD_ID to AOSP tag" map

config/build-index/build-id-to-tag.yml

@@ -0,0 +1,72 @@
+TP1A.220624.014: android-13.0.0_r1
+TP1A.220624.021: android-13.0.0_r2
+TP1A.220624.021.A1: android-13.0.0_r3
+TP1A.220905.004: android-13.0.0_r4
+TP1A.220905.004.A1: android-13.0.0_r5
+TP1A.220905.004.A2: android-13.0.0_r6
+TP1A.221005.002: android-13.0.0_r7
+TP1A.221005.003: android-13.0.0_r8
+TD1A.220804.009.A2: android-13.0.0_r9
+TD1A.220804.009.A5: android-13.0.0_r10
+TD1A.220804.031: android-13.0.0_r11
+TP1A.221105.002: android-13.0.0_r12
+TD1A.221105.001: android-13.0.0_r13
+TD1A.221105.003: android-13.0.0_r14
+TD1A.221105.001.A1: android-13.0.0_r15
+TQ1A.221205.011: android-13.0.0_r16
+TQ1A.221205.011.B1: android-13.0.0_r17
+TQ1A.221205.012: android-13.0.0_r18
+TQ1A.230105.001: android-13.0.0_r19
+TQ1A.230105.001.A2: android-13.0.0_r20
+TQ1A.230105.001.A3: android-13.0.0_r21
+TQ1A.230105.001.B1: android-13.0.0_r22
+TQ1A.230105.002: android-13.0.0_r23
+TQ1A.230105.002.A1: android-13.0.0_r24
+TQ1A.230205.001.A2: android-13.0.0_r27
+TQ1A.230205.001.B2: android-13.0.0_r28
+TQ1A.230205.001.D2: android-13.0.0_r29
+TQ1A.230205.002: android-13.0.0_r30
+TP1A.221005.002.B2: android-13.0.0_r31
+TQ2A.230305.008: android-13.0.0_r32
+TQ2A.230305.008.A1: android-13.0.0_r33
+TQ2A.230305.008.A3: android-13.0.0_r34
+TQ2A.230305.008.C1: android-13.0.0_r35
+TQ2A.230305.008.E1: android-13.0.0_r36
+TQ2A.230305.008.F1: android-13.0.0_r37
+TQ2A.230405.003: android-13.0.0_r38
+TQ2A.230405.003.A2: android-13.0.0_r39
+TQ2A.230405.003.B2: android-13.0.0_r40
+TQ2A.230405.003.E1: android-13.0.0_r41
+TQ2A.230405.003.G1: android-13.0.0_r42
+TQ2A.230505.002: android-13.0.0_r43
+TQ2A.230505.002.A1: android-13.0.0_r44
+TQ2A.230505.002.G1: android-13.0.0_r45
+TD4A.221205.042: android-13.0.0_r46
+TD4A.221205.042.A1: android-13.0.0_r47
+TD4A.221205.042.B1: android-13.0.0_r48
+TQ2B.230505.005.A1: android-13.0.0_r49
+TQ3A.230605.010: android-13.0.0_r50
+TQ3A.230605.011: android-13.0.0_r51
+TQ3A.230605.012: android-13.0.0_r52
+TQ3A.230605.010.A1: android-13.0.0_r53
+TQ3A.230605.012.A1: android-13.0.0_r54
+TD2A.230203.028: android-13.0.0_r55
+TQ3A.230605.009.A1: android-13.0.0_r56
+TD3A.230203.070.A1: android-13.0.0_r57
+TQ3C.230605.010.C1: android-13.0.0_r58
+TQ3C.230605.010.C2: android-13.0.0_r59
+TQ3C.230605.010.D1: android-13.0.0_r60
+TQ3A.230705.001: android-13.0.0_r61
+TQ3A.230705.001.A1: android-13.0.0_r62
+TQ3A.230705.001.B4: android-13.0.0_r63
+TQ3C.230705.001.B1: android-13.0.0_r64
+TQ3C.230705.001.C1: android-13.0.0_r65
+TQ3C.230705.001.C2: android-13.0.0_r66
+TQ3A.230805.001: android-13.0.0_r67
+TQ3A.230805.001.A1: android-13.0.0_r68
+TQ3A.230805.001.A2: android-13.0.0_r69
+TQ3A.230805.001.A3: android-13.0.0_r70
+TQ3A.230805.001.B1: android-13.0.0_r71
+TQ3C.230805.001.A3: android-13.0.0_r72
+TQ3C.230805.001.A4: android-13.0.0_r73
+TQ3C.230805.001.B2: android-13.0.0_r74

src/commands/update-aosp-tag-index.ts

@@ -0,0 +1,121 @@
+import { Command, flags } from '@oclif/command'
+import assert from 'assert'
+import { writeFileSync } from 'fs'
+import YAML from 'yaml'
+
+import { ADEVTOOL_DIR, BUILD_ID_TO_TAG_FILE } from '../config/paths'
+import { showGitDiff } from '../util/cli'
+import { readFile } from '../util/fs'
+import { GitLsRemote } from '../util/git'
+import { yamlStringifyNoFold } from '../util/yaml'
+
+type BuildIdToTag = Map<string, string>
+
+export async function loadBuildIdToTagMap(): Promise<BuildIdToTag | null> {
+  try {
+    let file = await readFile(BUILD_ID_TO_TAG_FILE)
+    return YAML.parse(file, { mapAsMap: true }) as BuildIdToTag
+  } catch {
+    return null
+  }
+}
+
+export default class UpdateAospTagIndex extends Command {
+  static description = `update buildId-to-tag map in ${BUILD_ID_TO_TAG_FILE}`
+
+  static flags = {
+    dryRun: flags.boolean({
+      char: 'n',
+      description: 'skip file update',
+    }),
+  }
+
+  async run() {
+    let { flags } = this.parse(UpdateAospTagIndex)
+
+    // let bases = ['android-vts', 'android-security', 'android', 'android-platform', 'android-cts']
+    let bases = ['android']
+    // let versions = ['11.0.0', '12.0.0', '12.1.0', '13.0.0']
+    let versions = ['13.0.0', '14.0.0']
+
+    let prefixes: string[] = []
+    bases.forEach(base =>
+      versions.forEach(version => {
+        if (base === 'android-cts' || base === 'android-vts') {
+          version = version.slice(0, -2)
+        }
+        prefixes.push(`${base}-${version}_r`)
+      }),
+    )
+
+    let map = (await loadBuildIdToTagMap()) ?? new Map<string, string>()
+    let lsRemote = await GitLsRemote.get('https://github.com/aosp-mirror/platform_build')
+
+    let knownTags = new Set(map.values())
+    let requests = new Map<string, Promise<Response>>()
+
+    for (let tagName of lsRemote.tags.keys()) {
+      if (knownTags.has(tagName)) {
+        continue
+      }
+
+      if (prefixes.find(p => tagName.startsWith(p)) === undefined) {
+        continue
+      }
+
+      let url = `https://raw.githubusercontent.com/aosp-mirror/platform_build/${tagName}/core/build_id.mk`
+      requests.set(tagName, fetch(url))
+    }
+
+    let numNewTags = 0
+
+    for (let [tagName, responsePromise] of requests.entries()) {
+      let resp = await responsePromise
+      assert(resp.ok, tagName)
+
+      let build_id_mk = await resp.text()
+      let buildIdPrefix = 'BUILD_ID='
+      let line = build_id_mk.split('\n').find(s => s.startsWith(buildIdPrefix))
+      assert(line !== undefined, 'missing BUILD_ID line in:\n' + build_id_mk)
+      let build_id = line.substring(buildIdPrefix.length).trim()
+
+      this.log(tagName + ': ' + build_id)
+      map.set(build_id, tagName)
+      ++numNewTags
+    }
+
+    if (!flags.dryRun && numNewTags > 0) {
+      let entries = Array.from(map.entries())
+        .sort(([_k1, tag1], [_k2, tag2]) => compareTags(tag1, tag2))
+
+      writeFileSync(BUILD_ID_TO_TAG_FILE, yamlStringifyNoFold(new Map(entries)))
+    }
+
+    if (numNewTags === 0) {
+      this.log('No new tags')
+      process.exit(1)
+    }
+
+    showGitDiff(ADEVTOOL_DIR, BUILD_ID_TO_TAG_FILE)
+  }
+}
+
+function getTagRevision(tag: string) {
+  let idx = tag.lastIndexOf('_r')
+  assert(idx > 0, tag)
+  return parseInt(tag.substring(idx + 2))
+}
+
+function getTagBase(tag: string) {
+  let idx = tag.lastIndexOf('_r')
+  assert(idx > 0, tag)
+  return tag.slice(0, idx)
+}
+
+function compareTags(a: string, b: string) {
+  let res = getTagBase(a).localeCompare(getTagBase(b))
+  if (res == 0) {
+    res = getTagRevision(a) - getTagRevision(b)
+  }
+  return res
+}

src/config/paths.ts

@@ -21,3 +21,5 @@ export const BUILD_INDEX_FILE = path.join(BUILD_INDEX_DIR, 'build-index.yml')
 export const MAIN_BUILD_INDEX_PART = path.join(BUILD_INDEX_DIR, 'build-index-main.yml')
 
 export const IMAGE_DOWNLOAD_DIR = process.env['ADEVTOOL_IMG_DOWNLOAD_DIR'] ?? path.join(ADEVTOOL_DIR, 'dl')
+
+export const BUILD_ID_TO_TAG_FILE = path.join(BUILD_INDEX_DIR, 'build-id-to-tag.yml')

src/util/git.ts

@@ -0,0 +1,39 @@
+import assert from 'assert'
+import { spawnAsync } from './process'
+
+export class GitLsRemote {
+  tags = new Map<string, string>()
+  branches = new Map<string, string>()
+
+  static async get(repoUrl: string) {
+    let lsRemote = await spawnAsync('git', ['ls-remote', '--tags', '--heads', repoUrl])
+
+    let result = new GitLsRemote()
+
+    const tagPrefix = 'refs/tags/'
+    const tagSuffix = '^{}'
+    const branchPrefix = 'refs/heads/'
+
+    for (let line of (await lsRemote).split('\n')) {
+      if (line.length === 0) {
+        continue
+      }
+      let split = line.split('\t')
+      assert(split.length == 2, line)
+      let commit = split[0]
+      let desc = split[1]
+
+      if (desc.startsWith(tagPrefix)) {
+        if (!desc.endsWith(tagSuffix)) {
+          continue
+        }
+        result.tags.set(desc.slice(tagPrefix.length, - tagSuffix.length), commit)
+      } else {
+        assert(desc.startsWith(branchPrefix), line)
+        result.branches.set(desc, commit)
+      }
+    }
+
+    return result
+  }
+}