From 55c6f0db6b5b1f229988f45905de1732a0346f09 Mon Sep 17 00:00:00 2001 From: Dmitry Muhomor Date: Mon, 28 Aug 2023 18:12:06 +0300 Subject: [PATCH] rewrite aapt2-based overlay parser Previous parser relied on text output of `aapt2/ dump xmltree` and `aapt2 dump resources` commands, which is not meant for parsing. This led to multiple parsing bugs: cutoff multiline strings, floating point parsing issues, quoting issues, non-English locale issues etc. Rewritten parser instead relies on protobuf aapt2 output, which allows to resolve most of these issues. A simple patch to aapt2 in frameworks/base is required to slightly expand its protobuf output. Other notable changes: - support for the "fraction" resource type - parsing is now parallelized at APK level --- src/blobs/overlays.ts | 529 ++++++++++++++++++++++++++---------------- 1 file changed, 328 insertions(+), 201 deletions(-) diff --git a/src/blobs/overlays.ts b/src/blobs/overlays.ts index d6cae5b..c6c0e05 100644 --- a/src/blobs/overlays.ts +++ b/src/blobs/overlays.ts @@ -1,21 +1,19 @@ +import assert from 'assert' import { promises as fs } from 'fs' +import { tmpdir } from 'os' import path from 'path' +import { unzip, ZipEntry } from 'unzipit' import xml2js from 'xml2js' import { serializeBlueprint } from '../build/soong' -import { aapt2 } from '../util/process' +import { Filters, filterValue } from '../config/filters' +import { Configuration } from '../proto-ts/frameworks/base/tools/aapt2/Configuration' +import { Item, ResourceTable, XmlAttribute, XmlNode } from '../proto-ts/frameworks/base/tools/aapt2/Resources' import { exists, listFilesRecursive } from '../util/fs' import { XML_HEADER } from '../util/headers' -import { parseLines } from '../util/parse' import { EXT_PARTITIONS } from '../util/partitions' -import { Filters, filterValue } from '../config/filters' - -const TARGET_PACKAGE_PATTERN = makeManifestRegex('targetPackage') -const TARGET_NAME_PATTERN = makeManifestRegex('targetName') - -// This is terrible, but aapt2 doesn't escape strings properly and some of these -// strings contain double quotes, which break our parser. -const EXCLUDE_LOCALES = new Set(['ar', 'iw']) +import { spawnAsyncNoOut } from '../util/process' +import { NodeFileReader } from '../util/zip' // Diff exclusions const DIFF_EXCLUDE_TYPES = new Set(['raw', 'xml', 'color']) @@ -43,15 +41,6 @@ export type ResValues = Map export type PartResValues = { [part: string]: ResValues } -function makeManifestRegex(attr: string) { - return new RegExp( - /^\s+A: http:\/\/schemas.android.com\/apk\/res\/android:/.source + - attr + - /\(0x[a-z0-9]+\)="(.+)" \(Raw: ".*$/.source, - 'm', // multiline flag - ) -} - function encodeResKey(key: ResKey) { // pkg/name:type/key|flags return ( @@ -91,195 +80,42 @@ function toResKey( }) } -function finishArray( - values: Map, - targetPkg: string, - targetName: string | null, - type: string | null, - key: string | null, - flags: string | null, - arrayLines: Array | null, -) { - // Exclude problematic locales and types (ID references) - let rawValue = arrayLines!.join('\n') - if (EXCLUDE_LOCALES.has(flags!) || rawValue.startsWith('[@0x')) { - return - } - - let array = parseAaptJson(rawValue) as Array - - // Change to typed array? - if (typeof array[0] === 'string') { - type = 'string-array' - } else if (typeof array[0] === 'number') { - // Float arrays are just , so check for integers - if (array.find(v => !Number.isInteger(v)) == undefined) { - type = 'integer-array' - } - } - - values.set(toResKey(targetPkg, targetName, type, key, flags), array) -} - -function parseAaptJson(value: string) { - // Fix backslash escapes - value = value.replaceAll(/\\/g, '\\\\') - - // Parse hex arrays - value = value.replaceAll(/\b0x[0-9a-f]+\b/g, value => `${parseInt(value.slice(2), 16)}`) - - return JSON.parse(value) -} - -function parseRsrcLines(rsrc: string, targetPkg: string, targetName: string | null) { - // Finished values with encoded res keys - let values: ResValues = new Map() - - // Current resource state machine - let curType: string | null = null - let curKey: string | null = null - let curFlags: string | null = null - let curArray: Array | null = null - - // Parse line-by-line - for (let line of parseLines(rsrc)) { - // Start resource - let resStart = line.match(/^resource 0x[a-z0-9]+ (.+)$/) - if (resStart) { - // Finish last array? - if (curArray != null) { - finishArray(values, targetPkg, targetName, curType, curKey, curFlags, curArray) - } - - let keyParts = resStart[1]!.split('/') - curType = keyParts[0] - curKey = keyParts[1] - curFlags = null - curArray = null - continue - } - - // New resource is array - let arrayLine = line.match(/^\(([a-zA-Z0-9\-_+]*)\) \(array\) size=\d+$/) - if (arrayLine) { - // Finish last array? - if (curArray != null) { - finishArray(values, targetPkg, targetName, curType, curKey, curFlags, curArray) - } - - // Start new array - curFlags = arrayLine[1] - curArray = [] - continue - } - - // New value - let valueLine = line.match(/^\(([a-zA-Z0-9\-_+]*)\) (.+)$/) - if (valueLine) { - curFlags = valueLine![1] - - // Exclude broken locales and styles for now - if (EXCLUDE_LOCALES.has(curFlags!) || curType == 'style') { - continue - } - - let value: ResValue - let rawValue = valueLine![2] - if (rawValue.startsWith('(file) ')) { - continue - } else if (curType == 'dimen') { - // Keep dimensions as strings to preserve unit - value = rawValue - } else if (curType == 'color') { - // Raw hex code - value = rawValue - } else if (rawValue.startsWith('0x')) { - // Hex integer - value = parseInt(rawValue.slice(2), 16) - } else if (rawValue.startsWith('(styled string) ')) { - // Skip styled strings for now - continue - } else if (curType == 'string') { - // Don't rely on quotes for simple strings - value = rawValue.slice(1, -1) - } else { - value = parseAaptJson(rawValue) - } - - values.set(toResKey(targetPkg, targetName, curType, curKey, curFlags), value) - } - - // New type section - let typeLine = line.match(/^type .+$/) - if (typeLine) { - // Just skip this line. Next resource/end will finish the last array, and this - // shouldn't be added to the last array. - continue - } - - // Continuation of array? - if (curArray != null) { - curArray.push(line) - } - } - - // Finish remaining array? - if (curArray != null) { - finishArray(values, targetPkg, targetName, curType, curKey, curFlags, curArray) - } - - return values -} - async function parseOverlayApksRecursive( aapt2Path: string, + partition: string, overlaysDir: string, pathCallback?: (path: string) => void, filters: Filters | null = null, ) { let values: ResValues = new Map() - for await (let apkPath of listFilesRecursive(overlaysDir)) { - if (path.extname(apkPath) != '.apk') { - continue + let tmpDir = fs.mkdtemp(path.join(tmpdir(), 'adevtool-overlay-parsing-')) + try { + let promises: Promise[] = [] + + for await (let apkPath of listFilesRecursive(overlaysDir)) { + if (path.extname(apkPath) != '.apk') { + continue + } + + if (filters != null && !filterValue(filters, path.relative(overlaysDir, apkPath))) { + continue + } + + promises.push(loadApkResValues(aapt2Path, partition, apkPath, await tmpDir)) } - if (pathCallback != undefined) { - pathCallback(apkPath) - } - - if (filters != null && !filterValue(filters, path.relative(overlaysDir, apkPath))) { - continue - } - - // Check the manifest for eligibility first - let manifest = await aapt2(aapt2Path, 'dump', 'xmltree', '--file', 'AndroidManifest.xml', apkPath) - // Overlays that have categories are user-controlled, so they're not relevant here - if (manifest.includes('A: http://schemas.android.com/apk/res/android:category(')) { - continue - } - // Prop-guarded overlays are almost always in AOSP already, so don't bother checking them - if (manifest.includes('A: http://schemas.android.com/apk/res/android:requiredSystemPropertyName(')) { - continue - } - - // Get the target package - let match = manifest.match(TARGET_PACKAGE_PATTERN) - if (!match) throw new Error(`Overlay ${apkPath} is missing target package`) - let targetPkg = match[1] - - // Get the target overlayable config name, if it exists - match = manifest.match(TARGET_NAME_PATTERN) - let targetName = match == undefined ? null : match[1] - - // Overlay is eligible, now read the resource table - let rsrc = await aapt2(aapt2Path, 'dump', 'resources', apkPath) - let apkValues = parseRsrcLines(rsrc, targetPkg, targetName) - - // Merge overlayed values - for (let [key, value] of apkValues) { - values.set(key, value) + for await (let resValues of promises) { + if (resValues === null) { + continue + } + // Merge overlayed values + for (let [key, value] of resValues) { + values.set(key, value) + } } + } finally { + await fs.rm(await tmpDir, { recursive: true }) } return values @@ -299,7 +135,7 @@ export async function parsePartOverlayApks( continue } - partValues[partition] = await parseOverlayApksRecursive(aapt2Path, src, pathCallback, filters) + partValues[partition] = await parseOverlayApksRecursive(aapt2Path, partition, src, pathCallback, filters) } return partValues @@ -447,6 +283,14 @@ export async function serializePartOverlays(partValues: PartResValues, overlaysD }, } as { [key: string]: any } + if (type === 'string') { + let s = value as string + if (s.match(/([@?\n\t'"])/)) { + // quote strings that have special characters + value = '"' + (value as string).replace('"', '\\"') + '"' + } + } + if (type.includes('array')) { entry.item = (value as Array).map(v => JSON.stringify(v)) } else { @@ -466,9 +310,12 @@ export async function serializePartOverlays(partValues: PartResValues, overlaysD let overlayDir = `${overlaysDir}/${partition}_${genTarget}` let resDir = `${overlayDir}/res/values` await fs.mkdir(resDir, { recursive: true }) - await fs.writeFile(`${overlayDir}/Android.bp`, bp) - await fs.writeFile(`${overlayDir}/AndroidManifest.xml`, manifest) - await fs.writeFile(`${resDir}/values.xml`, valuesXml) + let writes = [ + fs.writeFile(`${overlayDir}/Android.bp`, bp), + fs.writeFile(`${overlayDir}/AndroidManifest.xml`, manifest), + fs.writeFile(`${resDir}/values.xml`, valuesXml), + ] + await Promise.all(writes) buildPkgs.push(rroName) } @@ -476,3 +323,283 @@ export async function serializePartOverlays(partValues: PartResValues, overlaysD return buildPkgs } + +async function loadApkResValues(aapt2Path: string, partition: string, apkPath: string, tmpDir: string) { + let protoApk = path.join(tmpDir, partition + '_' + path.basename(apkPath, '.apk') + '.proto.apk') + // convert resource table and XMLs from binary to proto format + await spawnAsyncNoOut(aapt2Path, [ + 'convert', + '--for-adevtool', + '--output-format', + 'proto', + apkPath, + '-v', // verbose logging + '-o', + protoApk, + ]) + + let reader = new NodeFileReader(protoApk) + try { + let { entries: zipEntries } = await unzip(reader) + + let manifest = XmlNode.decode(await zipEntryAsUint8Array(zipEntries['AndroidManifest.xml'])) + + let overlayNode = manifest.element!.child.find(c => c.element?.name === 'overlay')! + + let namespaceUri = 'http://schemas.android.com/apk/res/android' + + let overlayAttrs = new Map() + + for (let a of overlayNode.element!.attribute) { + if (a.namespaceUri === namespaceUri) { + overlayAttrs.set(a.name, a) + } + } + + if (overlayAttrs.has('category')) { + // comment from original impl: + // Overlays that have categories are user-controlled, so they're not relevant here + return null + } + + if (overlayAttrs.has('requiredSystemPropertyName')) { + // comment from original impl: + // Prop-guarded overlays are almost always in AOSP already, so don't bother checking them + return null + } + + let targetPkgAttr = overlayAttrs.get('targetPackage') + assert(targetPkgAttr !== undefined, 'missing targetPkg overlay attribute for ' + apkPath) + let targetNameAttr = overlayAttrs.get('targetName') + + let targetPkg = targetPkgAttr.value + let targetName = targetNameAttr?.value ?? null + + let resourceTable = ResourceTable.decode(await zipEntryAsUint8Array(zipEntries['resources.pb'])) + + return resourseTableToResValues(resourceTable, targetPkg, targetName) + } finally { + reader.close() + await fs.rm(protoApk) + } +} + +function resourseTableToResValues(resTable: ResourceTable, targetPkg: string, targetName: string | null): ResValues { + let values: ResValues = new Map() + + for (let pkg of resTable.package) { + for (let resType of pkg.type) { + for (let resEntry of resType.entry) { + for (let resConfigValue of resEntry.configValue) { + let resConfig = resConfigValue.config! + let val = resConfigValue.value! + + let resItem = val.item + if (resItem !== undefined) { + let rv = protoResItemToResValue(resItem) + if (rv !== null) { + let resKey = toResKey(targetPkg, targetName, resType.name, resEntry.name, resConfigStr(resConfig)) + values.set(resKey, rv) + } + } else { + let cv = val.compoundValue! + if (cv.array) { + let items = cv.array.element.map(e => protoResItemToResValue(e.item!)) + + // check that all array entries are of supported type + if (items.find(i => i === null) === undefined) { + let typeName = 'array' + let firstItem = cv.array!.element[0]?.item + if (firstItem?.str !== undefined) { + typeName = 'string-array' + } else if ( + firstItem?.prim?.intHexadecimalValue !== undefined || + firstItem?.prim?.intDecimalValue !== undefined + ) { + typeName = 'integer-array' + } + let resKey = toResKey(targetPkg, targetName, typeName, resEntry.name, resConfigStr(resConfig)) + values.set(resKey, items as ResValue[]) + } + } + // TODO add support for more CompoundValue types + } + } + } + } + } + return values +} + +function protoResItemToResValue(item: Item): ResValue | null { + if (item.str !== undefined) { + return item.str.value + } + + if (item.prim) { + let p = item.prim + if (p.booleanValue !== undefined) { + return p.booleanValue + } + if (p.intDecimalValue !== undefined) { + return p.intDecimalValue + } + if (p.intHexadecimalValue !== undefined) { + return p.intHexadecimalValue + } + if (p.floatValue !== undefined) { + return roundFloat(p.floatValue) + } + if (p.emptyValue !== undefined) { + return '' + } + if (p.fractionValue !== undefined) { + return new Complex(ComplexType.TYPE_FRACTION, p.fractionValue).asString() + } + if (p.dimensionValue !== undefined) { + return new Complex(ComplexType.TYPE_DIMENSION, p.dimensionValue).asString() + } + if (p.colorArgb4Value !== undefined) { + return '#' + p.colorArgb4Value.toString(16) + } + if (p.colorArgb8Value !== undefined) { + return '#' + p.colorArgb8Value.toString(16) + } + if (p.colorRgb4Value !== undefined) { + return '#' + p.colorRgb4Value.toString(16) + } + if (p.colorRgb8Value !== undefined) { + return '#' + p.colorRgb8Value.toString(16) + } + if (p.nullValue !== undefined) { + return '' + } + } + // TODO add support for more types + + return null +} + +async function zipEntryAsUint8Array(e: ZipEntry) { + return new Uint8Array(await e.arrayBuffer()) +} + +enum ComplexType { + TYPE_DIMENSION, + TYPE_FRACTION, +} + +// configuration values joined by dashes, same format as in values- Android resource dirs (e.g. en-sw600dp-night) +function resConfigStr(c: Configuration) { + let v = c.stringified! + if (v === '') { + return null + } + return v +} + +function roundFloat(v: number) { + return parseFloat( + v.toLocaleString(undefined, { + useGrouping: false, + minimumFractionDigits: 0, + maximumFractionDigits: 5, + }), + ) +} + +// ported to TypeScript from frameworks/base/core/java/android/util/TypedValue.java +class Complex { + unit: number + + constructor(readonly type: ComplexType, readonly raw: number) { + this.unit = (raw >> COMPLEX_UNIT_SHIFT) & COMPLEX_UNIT_MASK + } + + asFloat() { + return ( + (this.raw & (COMPLEX_MANTISSA_MASK << COMPLEX_MANTISSA_SHIFT)) * RADIX_MULTS[(this.raw >> COMPLEX_RADIX_SHIFT) & COMPLEX_RADIX_MASK]) + } + + asString() { + let fv = this.asFloat() + let f = roundFloat(fv).toString() + switch (this.type) { + case ComplexType.TYPE_DIMENSION: { + switch (this.unit) { + case COMPLEX_UNIT_PX: + return f + 'px' + case COMPLEX_UNIT_DIP: + return f + 'dp' + case COMPLEX_UNIT_SP: + return f + 'sp' + case COMPLEX_UNIT_PT: + return f + 'pt' + case COMPLEX_UNIT_IN: + return f + 'in' + case COMPLEX_UNIT_MM: + return f + 'mm' + } + throw new Error('unknown unit ' + this.unit) + } + case ComplexType.TYPE_FRACTION: { + switch (this.unit) { + case COMPLEX_UNIT_FRACTION: + return roundFloat(fv * 100.0) + '%' + case COMPLEX_UNIT_FRACTION_PARENT: + return roundFloat(fv * 100.0) + '%p' + } + throw new Error('unknown unit ' + this.unit) + } + } + return null + } +} + +/** {@link #TYPE_DIMENSION} complex unit: Value is raw pixels. */ +const COMPLEX_UNIT_PX = 0 +/** {@link #TYPE_DIMENSION} complex unit: Value is Device Independent + * Pixels. */ +const COMPLEX_UNIT_DIP = 1 +/** {@link #TYPE_DIMENSION} complex unit: Value is a scaled pixel. */ +const COMPLEX_UNIT_SP = 2 +/** {@link #TYPE_DIMENSION} complex unit: Value is in points. */ +const COMPLEX_UNIT_PT = 3 +/** {@link #TYPE_DIMENSION} complex unit: Value is in inches. */ +const COMPLEX_UNIT_IN = 4 +/** {@link #TYPE_DIMENSION} complex unit: Value is in millimeters. */ +const COMPLEX_UNIT_MM = 5 + +/** {@link #TYPE_FRACTION} complex unit: A basic fraction of the overall + * size. */ +const COMPLEX_UNIT_FRACTION = 0 +/** {@link #TYPE_FRACTION} complex unit: A fraction of the parent size. */ +const COMPLEX_UNIT_FRACTION_PARENT = 1 + +/** Complex data: where the radix information is, telling where the decimal + * place appears in the mantissa. */ +const COMPLEX_RADIX_SHIFT = 4 +/** Complex data: mask to extract radix information (after shifting by + * {@link #COMPLEX_RADIX_SHIFT}). This give us 4 possible fixed point + * representations as defined below. */ +const COMPLEX_RADIX_MASK = 0x3 + +/** Complex data: bit location of mantissa information. */ +const COMPLEX_MANTISSA_SHIFT = 8 +/** Complex data: mask to extract mantissa information (after shifting by + * {@link #COMPLEX_MANTISSA_SHIFT}). This gives us 23 bits of precision + * the top bit is the sign. */ +const COMPLEX_MANTISSA_MASK = 0xffffff + +const MANTISSA_MULT = 1.0 / (1<