From de6b18ea6bff570ef6631a8f185f9efab4752c5c Mon Sep 17 00:00:00 2001 From: Danny Lin Date: Sun, 7 Nov 2021 03:21:28 -0800 Subject: [PATCH] extract: Split into multiple files --- src/blobs/copy.ts | 36 +++++ src/blobs/entry.ts | 7 + src/blobs/file_list.ts | 44 ++++++ src/build/make.ts | 28 ++++ src/build/soong.ts | 215 +++++++++++++++++++++++++++ src/commands/extract.ts | 319 +++------------------------------------- 6 files changed, 354 insertions(+), 295 deletions(-) create mode 100644 src/blobs/copy.ts create mode 100644 src/blobs/entry.ts create mode 100644 src/blobs/file_list.ts create mode 100644 src/build/make.ts create mode 100644 src/build/soong.ts diff --git a/src/blobs/copy.ts b/src/blobs/copy.ts new file mode 100644 index 0000000..68a3cd2 --- /dev/null +++ b/src/blobs/copy.ts @@ -0,0 +1,36 @@ +import {promises as fs} from 'fs' +import * as path from 'path' +import * as chalk from 'chalk' +import * as ora from 'ora' + +import { BlobEntry } from './entry' + +export async function copyBlobs(entries: Array, srcDir: string, outDir: string) { + let spinner = ora({ + prefixText: chalk.bold(chalk.greenBright('Copying blobs')), + color: 'green', + }).start() + + for (let entry of entries) { + spinner.text = entry.srcPath + + let outPath = `${outDir}/${entry.srcPath}` + await fs.mkdir(path.dirname(outPath), {recursive: true}) + + // Some files need patching + let srcPath = `${srcDir}/${entry.srcPath}` + if (entry.path.endsWith('.xml')) { + let xml = await fs.readFile(srcPath, {encoding: 'utf8'}) + // Fix Qualcomm "version 2.0" XMLs + if (xml.startsWith(' a.srcPath.localeCompare(b.srcPath)) +} diff --git a/src/build/make.ts b/src/build/make.ts new file mode 100644 index 0000000..0b3ea40 --- /dev/null +++ b/src/build/make.ts @@ -0,0 +1,28 @@ +import { BlobEntry } from '../blobs/entry'; + +const CONT_SEPARATOR = ' \\\n ' + +export interface ProductMakefile { + namespaces: Array + copyFiles: Array + packages: Array +} + +export function blobToFileCopy(entry: BlobEntry, proprietaryDir: string) { + let copyPart = entry.partition.toUpperCase() + return `${proprietaryDir}/${entry.srcPath}:$(TARGET_COPY_OUT_${copyPart})/${entry.path}` +} + +export function serializeProductMakefile(makefile: ProductMakefile) { + return `# Generated by adevtool, do not edit + +PRODUCT_SOONG_NAMESPACES += \\ + ${makefile.namespaces.join(CONT_SEPARATOR)} + +PRODUCT_COPY_FILES += \\ + ${makefile.copyFiles.join(CONT_SEPARATOR)} + +PRODUCT_PACKAGES += \\ + ${makefile.packages.join(CONT_SEPARATOR)} +` +} diff --git a/src/build/soong.ts b/src/build/soong.ts new file mode 100644 index 0000000..de2f31a --- /dev/null +++ b/src/build/soong.ts @@ -0,0 +1,215 @@ +import * as util from 'util' + +import { BlobEntry } from '../blobs/entry' + +export interface TargetSrcs { + srcs: Array +} + +export interface SharedLibraryModule { + strip: { + none: boolean + } + target: { + android_arm?: TargetSrcs + android_arm64?: TargetSrcs + } + compile_multilib: string + check_elf_files: boolean + prefer: boolean +} + +export interface ApkModule { + apk: string + certificate?: string + presigned?: boolean + privileged?: boolean + dex_preopt: { + enabled: boolean + } +} + +export interface JarModule { + jars: Array +} + +export interface EtcXmlModule { + src: string + filename_from_src: boolean + sub_dir: string +} + +export type SoongModuleSpecific = { + // Type is mandatory initially, but this is deleted for serialization + _type?: string +} & ( + SharedLibraryModule | + ApkModule | + JarModule | + EtcXmlModule +) + +export type SoongModule = { + name: string + owner: string + system_ext_specific?: boolean + product_specific?: boolean + soc_specific?: boolean +} & SoongModuleSpecific + +export function blobToSoongModule( + name: string, + ext: string, + vendor: string, + entry: BlobEntry, + entrySrcPaths: Set, +) { + // Module name = file name + let moduleSrcPath = `proprietary/${entry.srcPath}` + + // Type and info is based on file extension + let moduleSpecific: SoongModuleSpecific + if (ext == '.so') { + // Extract architecture from lib dir + let pathParts = entry.srcPath.split('/') + let libDir = pathParts.at(-2) + let curArch: string + if (libDir == 'lib') { + curArch = '32' + } else { + // Assume 64-bit native if not lib/lib64 + curArch = '64' + } + let arch = curArch + + // Check for the other arch + let otherLibDir = arch == '32' ? 'lib64' : 'lib' + let otherSrcPath = [ + // Preceding parts + ...pathParts.slice(0, -2), + // lib / lib64 + otherLibDir, + // Trailing part (file name) + pathParts.at(-1), + ].join('/') + if (entrySrcPaths.has(otherSrcPath)) { + // Both archs are present + arch = 'both' + } + + // For single-arch + let targetSrcs = { + srcs: [moduleSrcPath], + } as TargetSrcs + + // For multi-arch + let targetSrcs32 = (curArch == '32') ? targetSrcs : { + srcs: [`proprietary/${otherSrcPath}`], + } as TargetSrcs + let targetSrcs64 = (curArch == '64') ? targetSrcs : { + srcs: [`proprietary/${otherSrcPath}`], + } as TargetSrcs + + moduleSpecific = { + _type: 'cc_prebuilt_library_shared', + strip: { + none: true, + }, + target: { + ...(arch == '32' && { android_arm: targetSrcs }), + ...(arch == '64' && { android_arm64: targetSrcs }), + ...(arch == 'both' && { + android_arm: targetSrcs32, + android_arm64: targetSrcs64, + }), + }, + compile_multilib: arch, + check_elf_files: false, + prefer: true, + } + } else if (ext == '.apk') { + moduleSpecific = { + _type: 'android_app_import', + apk: moduleSrcPath, + ...(entry.isPresigned && { presigned: true } || { certificate: 'platform' }), + ...(entry.path.startsWith('priv-app/') && { privileged: true }), + dex_preopt: { + enabled: false, + }, + } + } else if (ext == '.jar') { + moduleSpecific = { + _type: 'dex_import', + jars: [moduleSrcPath], + } + } else if (ext == '.xml') { + // Only etc/ XMLs are supported for now + let pathParts = entry.path.split('/') + if (pathParts[0] != 'etc') { + throw new Error(`XML file ${entry.srcPath} is not in etc/`) + } + + moduleSpecific = { + _type: 'prebuilt_etc_xml', + src: moduleSrcPath, + filename_from_src: true, + sub_dir: pathParts.slice(1).join('/'), + } + } else { + throw new Error(`File ${entry.srcPath} has unknown extension ${ext}`) + } + + return { + name: name, + owner: vendor, + ...moduleSpecific, + + // Partition flag + ...(entry.partition == 'system_ext' && { system_ext_specific: true }), + ...(entry.partition == 'product' && { product_specific: true }), + ...(entry.partition == 'vendor' && { soc_specific: true }), + } as SoongModule +} + +export function serializeModule(module: SoongModule) { + // Type prepended to Soong module props, so remove it from the object + let type = module._type; + delete module._type; + + // Initial serialization pass. Node.js util.inspect happens to be identical to Soong format. + let serialized = util.inspect(module, { + depth: Infinity, + maxArrayLength: Infinity, + maxStringLength: Infinity, + }) + + // ' -> " + serialized = serialized.replaceAll("'", '"') + // 4-space indentation + serialized = serialized.replaceAll(' ', ' ') + // Prepend type + serialized = `${type} ${serialized}` + // Add trailing comma to last prop + let serialLines = serialized.split('\n') + serialLines[serialLines.length - 2] = serialLines.at(-2) + ',' + serialized = serialLines.join('\n') + + return serialized +} + +export function serializeBlueprint(modules: IterableIterator) { + // Soong pass 2: serialize module objects + let serializedModules = [] + for (let module of modules) { + let serialized = serializeModule(module) + serializedModules.push(serialized) + } + + return `// Generated by adevtool, do not edit + +soong_namespace { +} + +${serializedModules.join('\n\n')} +` +} diff --git a/src/commands/extract.ts b/src/commands/extract.ts index 96be9b7..de963e5 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -2,145 +2,12 @@ import {Command, flags} from '@oclif/command' import {promises as fs} from 'fs' import * as path from 'path' import * as chalk from 'chalk' -import * as ora from 'ora' -import * as util from 'util' -// Excluding system -const PARTITIONS = new Set(['system_ext', 'product', 'vendor']) - -interface BlobEntry { - partition: string - path: string - srcPath: string - isPresigned: boolean - isNamedDependency: boolean -} - -interface TargetSrcs { - srcs: Array -} - -interface PrebuiltLibraryModule { - strip: { - none: boolean - } - target: { - android_arm?: TargetSrcs - android_arm64?: TargetSrcs - } - compile_multilib: string - check_elf_files: boolean - prefer: boolean -} - -interface ApkModule { - apk: string - certificate?: string - presigned?: boolean - privileged?: boolean - dex_preopt: { - enabled: boolean - } -} - -interface JarModule { - jars: Array -} - -interface EtcXmlModule { - src: string - filename_from_src: boolean - sub_dir: string -} - -type ModuleSpecific = { - // Type is mandatory initially, but this is deleted for serialization - _type?: string -} & ( - PrebuiltLibraryModule | - ApkModule | - JarModule | - EtcXmlModule -) - -type Module = { - name: string - owner: string - system_ext_specific?: boolean - product_specific?: boolean - soc_specific?: boolean -} & ModuleSpecific - -async function parseList(listPath: string) { - let list = await fs.readFile(listPath, {encoding: 'utf8'}) - let entries = [] - - for (let line of list.split('\n')) { - // Ignore comments and empty/blank lines - if (line.length == 0 || line.startsWith('#') || line.match(/^\s*$/)) { - continue - } - - // Split into path and flags first, ignoring whitespace - let [srcPath, postModifiers] = line.trim().split(';') - let modifiers = (postModifiers ?? '').split('|') - - // Parse "named dependency" flag (preceding -) - let isNamedDependency = srcPath.startsWith('-') - if (isNamedDependency) { - srcPath = srcPath.slice(1) - } - - // Split path into partition and sub-partition path - let pathParts = srcPath.split('/') - let partition = pathParts[0] - if (!PARTITIONS.has(partition)) { - partition = 'system' - } - let path = pathParts.slice(1).join('/') - - entries.push({ - partition: partition, - path: path, - srcPath: srcPath, - isPresigned: modifiers.includes('PRESIGNED'), - isNamedDependency: isNamedDependency, - } as BlobEntry) - } - - // Sort by source path - return entries.sort((a, b) => a.srcPath.localeCompare(b.srcPath)) -} - -async function copyBlobs(entries: Array, srcDir: string, outDir: string) { - let spinner = ora({ - prefixText: chalk.bold(chalk.greenBright('Copying blobs')), - color: 'green', - }).start() - - for (let entry of entries) { - spinner.text = entry.srcPath - - let outPath = `${outDir}/${entry.srcPath}` - await fs.mkdir(path.dirname(outPath), {recursive: true}) - - // Some files need patching - let srcPath = `${srcDir}/${entry.srcPath}` - if (entry.path.endsWith('.xml')) { - let xml = await fs.readFile(srcPath, {encoding: 'utf8'}) - // Fix Qualcomm "version 2.0" XMLs - if (xml.startsWith(', @@ -149,11 +16,12 @@ async function generateBuild( outDir: string, proprietaryDir: string, ) { - // Fast lookup for libs + // Fast lookup for other arch libs let entrySrcPaths = new Set(entries.map(e => e.srcPath)) + // Create Soong modules and Make rules let copyFiles = [] - let namedModules = new Map() + let namedModules = new Map() for (let entry of entries) { if (entry.isNamedDependency) { // Named dependencies -> Soong blueprint @@ -161,172 +29,32 @@ async function generateBuild( // Module name = file name let ext = path.extname(entry.path) let name = path.basename(entry.path, ext) - let moduleSrcPath = `proprietary/${entry.srcPath}` // Skip if already done (e.g. other lib arch) if (namedModules.has(name)) { continue } - // Type and info is based on file extension - let moduleSpecific: ModuleSpecific - if (ext == '.so') { - // Extract architecture from lib dir - let pathParts = entry.srcPath.split('/') - let libDir = pathParts.at(-2) - let curArch: string - if (libDir == 'lib') { - curArch = '32' - } else { - // Assume 64-bit native if not lib/lib64 - curArch = '64' - } - let arch = curArch - - // Check for the other arch - let otherLibDir = arch == '32' ? 'lib64' : 'lib' - let otherSrcPath = [ - // Preceding parts - ...pathParts.slice(0, -2), - // lib / lib64 - otherLibDir, - // Trailing part (file name) - pathParts.at(-1), - ].join('/') - if (entrySrcPaths.has(otherSrcPath)) { - // Both archs are present - arch = 'both' - } - - // For single arch - let targetSrcs = { - srcs: [moduleSrcPath], - } as TargetSrcs - - // For multi arch - let targetSrcs32 = (curArch == '32') ? targetSrcs : { - srcs: [`proprietary/${otherSrcPath}`], - } as TargetSrcs - let targetSrcs64 = (curArch == '64') ? targetSrcs : { - srcs: [`proprietary/${otherSrcPath}`], - } as TargetSrcs - - moduleSpecific = { - _type: 'cc_prebuilt_library_shared', - strip: { - none: true, - }, - target: { - ...(arch == '32' && { android_arm: targetSrcs }), - ...(arch == '64' && { android_arm64: targetSrcs }), - ...(arch == 'both' && { - android_arm: targetSrcs32, - android_arm64: targetSrcs64, - }), - }, - compile_multilib: arch, - check_elf_files: false, - prefer: true, - } - } else if (ext == '.apk') { - moduleSpecific = { - _type: 'android_app_import', - apk: moduleSrcPath, - ...(entry.isPresigned && { presigned: true } || { certificate: 'platform' }), - ...(entry.path.startsWith('priv-app/') && { privileged: true }), - dex_preopt: { - enabled: false, - }, - } - } else if (ext == '.jar') { - moduleSpecific = { - _type: 'dex_import', - jars: [moduleSrcPath], - } - } else if (ext == '.xml') { - // Only etc/ XMLs are supported for now - let pathParts = entry.path.split('/') - if (pathParts[0] != 'etc') { - throw new Error(`XML file ${entry.srcPath} is not in etc/`) - } - - moduleSpecific = { - _type: 'prebuilt_etc_xml', - src: moduleSrcPath, - filename_from_src: true, - sub_dir: pathParts.slice(1).join('/'), - } - } else { - throw new Error(`File ${entry.srcPath} has unknown extension ${ext}`) - } - - let module = { - name: name, - owner: vendor, - ...moduleSpecific, - - // Partition flag - ...(entry.partition == 'system_ext' && { system_ext_specific: true }), - ...(entry.partition == 'product' && { product_specific: true }), - ...(entry.partition == 'vendor' && { soc_specific: true }), - } as Module - + let module = blobToSoongModule(name, ext, vendor, entry, entrySrcPaths) namedModules.set(name, module) } else { // Other files -> Kati Makefile // Simple PRODUCT_COPY_FILES line - let copyPart = entry.partition.toUpperCase() - copyFiles.push(`${proprietaryDir}/${entry.srcPath}:$(TARGET_COPY_OUT_${copyPart})/${entry.path}`) + copyFiles.push(blobToFileCopy(entry, proprietaryDir)) } } - // Soong pass 2: serialize module objects - let serializedModules = [] - for (let module of namedModules.values()) { - // Type prepended to Soong module props, so remove it from the object - let type = module._type; - delete module._type; - - // Initial serialization pass. Node.js util.inspect happens to be identical to Soong format. - let serialized = util.inspect(module, { - depth: Infinity, - maxArrayLength: Infinity, - maxStringLength: Infinity, - }) - - // ' -> " - serialized = serialized.replaceAll("'", '"') - // 4-space indentation - serialized = serialized.replaceAll(' ', ' ') - // Prepend type - serialized = `${type} ${serialized}` - // Add trailing comma to last prop - let serialLines = serialized.split('\n') - serialLines[serialLines.length - 2] = serialLines.at(-2) + ',' - serialized = serialLines.join('\n') - - serializedModules.push(serialized) - } - - let blueprint = `// Generated by adevtool, do not edit - -soong_namespace { -} - -${serializedModules.join('\n\n')} -` - - let makefile = `# Generated by adevtool, do not edit - -PRODUCT_SOONG_NAMESPACES += \\ - ${outDir} - -PRODUCT_COPY_FILES += \\ - ${copyFiles.join(' \\\n ')} -` - + // Serialize Soong blueprint + let blueprint = serializeBlueprint(namedModules.values()) fs.writeFile(`${outDir}/Android.bp`, blueprint) + + // Serialize product makefile + let makefile = serializeProductMakefile({ + namespaces: [outDir], + packages: Array.from(namedModules.keys()), + copyFiles: copyFiles, + }) fs.writeFile(`${outDir}/${device}-vendor.mk`, makefile) } @@ -340,14 +68,15 @@ export default class Extract extends Command { source: flags.string({char: 's', description: 'path to mounted factory images', required: true}), } - static args = [{name: 'list'}] + static args = [{name: 'listPath'}] async run() { - let {args: {list}, flags: {vendor, device, source}} = this.parse(Extract) + let {args: {listPath}, flags: {vendor, device, source}} = this.parse(Extract) // Parse list this.log(chalk.bold(chalk.greenBright('Parsing list'))) - let entries = await parseList(list) + let list = await fs.readFile(listPath, {encoding: 'utf8'}) + let entries = parseFileList(list) // Prepare output directories let outDir = `vendor/${vendor}/${device}` @@ -357,7 +86,7 @@ export default class Extract extends Command { await fs.mkdir(proprietaryDir, {recursive: true}) // Copy blobs - //await copyBlobs(entries, source, proprietaryDir) + await copyBlobs(entries, source, proprietaryDir) // Generate build files this.log(chalk.bold(chalk.greenBright('Generating build files')))