Blaster4385/adevtool
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

extract: Split into multiple files

Browse source
This commit is contained in:
Danny Lin 2021-11-07 03:21:28 -08:00
parent 9a7b9f8add
commit de6b18ea6b
6 changed files with 354 additions and 295 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
src/blobs/copy.ts Normal file
View file

@ -0,0 +1,36 @@
import {promises as fs} from 'fs'
import * as path from 'path'
import * as chalk from 'chalk'
import * as ora from 'ora'
import { BlobEntry } from './entry'
export async function copyBlobs(entries: Array<BlobEntry>, srcDir: string, outDir: string) {
let spinner = ora({
prefixText: chalk.bold(chalk.greenBright('Copying blobs')),
color: 'green',
}).start()
for (let entry of entries) {
spinner.text = entry.srcPath
let outPath = `${outDir}/${entry.srcPath}`
await fs.mkdir(path.dirname(outPath), {recursive: true})
// Some files need patching
let srcPath = `${srcDir}/${entry.srcPath}`
if (entry.path.endsWith('.xml')) {
let xml = await fs.readFile(srcPath, {encoding: 'utf8'})
// Fix Qualcomm "version 2.0" XMLs
if (xml.startsWith('<?xml version="2.0"')) {
let patched = xml.replace(/^<\?xml version="2.0"/, '<?xml version="1.0"')
await fs.writeFile(outPath, patched)
continue
}
}
await fs.copyFile(srcPath, outPath)
}
spinner.stopAndPersist()
}

7
src/blobs/entry.ts Normal file
View file

@ -0,0 +1,7 @@
export interface BlobEntry {
partition: string
path: string
srcPath: string
isPresigned: boolean
isNamedDependency: boolean
}

44
src/blobs/file_list.ts Normal file
View file

@ -0,0 +1,44 @@
import { BlobEntry } from './entry'
// Excluding system
const PARTITIONS = new Set(['system_ext', 'product', 'vendor'])
export function parseFileList(list: string) {
let entries = []
for (let line of list.split('\n')) {
// Ignore comments and empty/blank lines
if (line.length == 0 || line.startsWith('#') || line.match(/^\s*$/)) {
continue
}
// Split into path and flags first, ignoring whitespace
let [srcPath, postModifiers] = line.trim().split(';')
let modifiers = (postModifiers ?? '').split('|')
// Parse "named dependency" flag (preceding -)
let isNamedDependency = srcPath.startsWith('-')
if (isNamedDependency) {
srcPath = srcPath.slice(1)
}
// Split path into partition and sub-partition path
let pathParts = srcPath.split('/')
let partition = pathParts[0]
if (!PARTITIONS.has(partition)) {
partition = 'system'
}
let path = pathParts.slice(1).join('/')
entries.push({
partition: partition,
path: path,
srcPath: srcPath,
isPresigned: modifiers.includes('PRESIGNED'),
isNamedDependency: isNamedDependency,
} as BlobEntry)
}
// Sort by source path
return entries.sort((a, b) => a.srcPath.localeCompare(b.srcPath))
}

28
src/build/make.ts Normal file
View file

@ -0,0 +1,28 @@
import { BlobEntry } from '../blobs/entry';
const CONT_SEPARATOR = ' \\\n '
export interface ProductMakefile {
namespaces: Array<string>
copyFiles: Array<string>
packages: Array<string>
}
export function blobToFileCopy(entry: BlobEntry, proprietaryDir: string) {
let copyPart = entry.partition.toUpperCase()
return `${proprietaryDir}/${entry.srcPath}:$(TARGET_COPY_OUT_${copyPart})/${entry.path}`
}
export function serializeProductMakefile(makefile: ProductMakefile) {
return `# Generated by adevtool, do not edit
PRODUCT_SOONG_NAMESPACES += \\
${makefile.namespaces.join(CONT_SEPARATOR)}
PRODUCT_COPY_FILES += \\
${makefile.copyFiles.join(CONT_SEPARATOR)}
PRODUCT_PACKAGES += \\
${makefile.packages.join(CONT_SEPARATOR)}
`
}

215
src/build/soong.ts Normal file
View file

@ -0,0 +1,215 @@
import * as util from 'util'
import { BlobEntry } from '../blobs/entry'
export interface TargetSrcs {
srcs: Array<string>
}
export interface SharedLibraryModule {
strip: {
none: boolean
}
target: {
android_arm?: TargetSrcs
android_arm64?: TargetSrcs
}
compile_multilib: string
check_elf_files: boolean
prefer: boolean
}
export interface ApkModule {
apk: string
certificate?: string
presigned?: boolean
privileged?: boolean
dex_preopt: {
enabled: boolean
}
}
export interface JarModule {
jars: Array<string>
}
export interface EtcXmlModule {
src: string
filename_from_src: boolean
sub_dir: string
}
export type SoongModuleSpecific = {
// Type is mandatory initially, but this is deleted for serialization
_type?: string
} & (
SharedLibraryModule |
ApkModule |
JarModule |
EtcXmlModule
)
export type SoongModule = {
name: string
owner: string
system_ext_specific?: boolean
product_specific?: boolean
soc_specific?: boolean
} & SoongModuleSpecific
export function blobToSoongModule(
name: string,
ext: string,
vendor: string,
entry: BlobEntry,
entrySrcPaths: Set<string>,
) {
// Module name = file name
let moduleSrcPath = `proprietary/${entry.srcPath}`
// Type and info is based on file extension
let moduleSpecific: SoongModuleSpecific
if (ext == '.so') {
// Extract architecture from lib dir
let pathParts = entry.srcPath.split('/')
let libDir = pathParts.at(-2)
let curArch: string
if (libDir == 'lib') {
curArch = '32'
} else {
// Assume 64-bit native if not lib/lib64
curArch = '64'
}
let arch = curArch
// Check for the other arch
let otherLibDir = arch == '32' ? 'lib64' : 'lib'
let otherSrcPath = [
// Preceding parts
...pathParts.slice(0, -2),
// lib / lib64
otherLibDir,
// Trailing part (file name)
pathParts.at(-1),
].join('/')
if (entrySrcPaths.has(otherSrcPath)) {
// Both archs are present
arch = 'both'
}
// For single-arch
let targetSrcs = {
srcs: [moduleSrcPath],
} as TargetSrcs
// For multi-arch
let targetSrcs32 = (curArch == '32') ? targetSrcs : {
srcs: [`proprietary/${otherSrcPath}`],
} as TargetSrcs
let targetSrcs64 = (curArch == '64') ? targetSrcs : {
srcs: [`proprietary/${otherSrcPath}`],
} as TargetSrcs
moduleSpecific = {
_type: 'cc_prebuilt_library_shared',
strip: {
none: true,
},
target: {
...(arch == '32' && { android_arm: targetSrcs }),
...(arch == '64' && { android_arm64: targetSrcs }),
...(arch == 'both' && {
android_arm: targetSrcs32,
android_arm64: targetSrcs64,
}),
},
compile_multilib: arch,
check_elf_files: false,
prefer: true,
}
} else if (ext == '.apk') {
moduleSpecific = {
_type: 'android_app_import',
apk: moduleSrcPath,
...(entry.isPresigned && { presigned: true } || { certificate: 'platform' }),
...(entry.path.startsWith('priv-app/') && { privileged: true }),
dex_preopt: {
enabled: false,
},
}
} else if (ext == '.jar') {
moduleSpecific = {
_type: 'dex_import',
jars: [moduleSrcPath],
}
} else if (ext == '.xml') {
// Only etc/ XMLs are supported for now
let pathParts = entry.path.split('/')
if (pathParts[0] != 'etc') {
throw new Error(`XML file ${entry.srcPath} is not in etc/`)
}
moduleSpecific = {
_type: 'prebuilt_etc_xml',
src: moduleSrcPath,
filename_from_src: true,
sub_dir: pathParts.slice(1).join('/'),
}
} else {
throw new Error(`File ${entry.srcPath} has unknown extension ${ext}`)
}
return {
name: name,
owner: vendor,
...moduleSpecific,
// Partition flag
...(entry.partition == 'system_ext' && { system_ext_specific: true }),
...(entry.partition == 'product' && { product_specific: true }),
...(entry.partition == 'vendor' && { soc_specific: true }),
} as SoongModule
}
export function serializeModule(module: SoongModule) {
// Type prepended to Soong module props, so remove it from the object
let type = module._type;
delete module._type;
// Initial serialization pass. Node.js util.inspect happens to be identical to Soong format.
let serialized = util.inspect(module, {
depth: Infinity,
maxArrayLength: Infinity,
maxStringLength: Infinity,
})
// ' -> "
serialized = serialized.replaceAll("'", '"')
// 4-space indentation
serialized = serialized.replaceAll(' ', ' ')
// Prepend type
serialized = `${type} ${serialized}`
// Add trailing comma to last prop
let serialLines = serialized.split('\n')
serialLines[serialLines.length - 2] = serialLines.at(-2) + ','
serialized = serialLines.join('\n')
return serialized
}
export function serializeBlueprint(modules: IterableIterator<SoongModule>) {
// Soong pass 2: serialize module objects
let serializedModules = []
for (let module of modules) {
let serialized = serializeModule(module)
serializedModules.push(serialized)
}
return `// Generated by adevtool, do not edit
soong_namespace {
}
${serializedModules.join('\n\n')}
`
}

319
src/commands/extract.ts
View file

@ -2,145 +2,12 @@ import {Command, flags} from '@oclif/command'
import {promises as fs} from 'fs'
import * as path from 'path'
import * as chalk from 'chalk'
import * as ora from 'ora'
import * as util from 'util'
// Excluding system
const PARTITIONS = new Set(['system_ext', 'product', 'vendor'])
interface BlobEntry {
partition: string
path: string
srcPath: string
isPresigned: boolean
isNamedDependency: boolean
}
interface TargetSrcs {
srcs: Array<string>
}
interface PrebuiltLibraryModule {
strip: {
none: boolean
}
target: {
android_arm?: TargetSrcs
android_arm64?: TargetSrcs
}
compile_multilib: string
check_elf_files: boolean
prefer: boolean
}
interface ApkModule {
apk: string
certificate?: string
presigned?: boolean
privileged?: boolean
dex_preopt: {
enabled: boolean
}
}
interface JarModule {
jars: Array<string>
}
interface EtcXmlModule {
src: string
filename_from_src: boolean
sub_dir: string
}
type ModuleSpecific = {
// Type is mandatory initially, but this is deleted for serialization
_type?: string
} & (
PrebuiltLibraryModule |
ApkModule |
JarModule |
EtcXmlModule
)
type Module = {
name: string
owner: string
system_ext_specific?: boolean
product_specific?: boolean
soc_specific?: boolean
} & ModuleSpecific
async function parseList(listPath: string) {
let list = await fs.readFile(listPath, {encoding: 'utf8'})
let entries = []
for (let line of list.split('\n')) {
// Ignore comments and empty/blank lines
if (line.length == 0 || line.startsWith('#') || line.match(/^\s*$/)) {
continue
}
// Split into path and flags first, ignoring whitespace
let [srcPath, postModifiers] = line.trim().split(';')
let modifiers = (postModifiers ?? '').split('|')
// Parse "named dependency" flag (preceding -)
let isNamedDependency = srcPath.startsWith('-')
if (isNamedDependency) {
srcPath = srcPath.slice(1)
}
// Split path into partition and sub-partition path
let pathParts = srcPath.split('/')
let partition = pathParts[0]
if (!PARTITIONS.has(partition)) {
partition = 'system'
}
let path = pathParts.slice(1).join('/')
entries.push({
partition: partition,
path: path,
srcPath: srcPath,
isPresigned: modifiers.includes('PRESIGNED'),
isNamedDependency: isNamedDependency,
} as BlobEntry)
}
// Sort by source path
return entries.sort((a, b) => a.srcPath.localeCompare(b.srcPath))
}
async function copyBlobs(entries: Array<BlobEntry>, srcDir: string, outDir: string) {
let spinner = ora({
prefixText: chalk.bold(chalk.greenBright('Copying blobs')),
color: 'green',
}).start()
for (let entry of entries) {
spinner.text = entry.srcPath
let outPath = `${outDir}/${entry.srcPath}`
await fs.mkdir(path.dirname(outPath), {recursive: true})
// Some files need patching
let srcPath = `${srcDir}/${entry.srcPath}`
if (entry.path.endsWith('.xml')) {
let xml = await fs.readFile(srcPath, {encoding: 'utf8'})
// Fix Qualcomm "version 2.0" XMLs
if (xml.startsWith('<?xml version="2.0"')) {
let patched = xml.replace(/^<\?xml version="2.0"/, '<?xml version="1.0"')
await fs.writeFile(outPath, patched)
continue
}
}
await fs.copyFile(srcPath, outPath)
}
spinner.stopAndPersist()
}
import { blobToSoongModule, serializeBlueprint, SoongModule } from '../build/soong'
import { BlobEntry } from '../blobs/entry'
import { parseFileList } from '../blobs/file_list'
import { copyBlobs } from '../blobs/copy'
import { blobToFileCopy, serializeProductMakefile } from '../build/make'
async function generateBuild(
entries: Array<BlobEntry>,
@ -149,11 +16,12 @@ async function generateBuild(
outDir: string,
proprietaryDir: string,
) {
// Fast lookup for libs
// Fast lookup for other arch libs
let entrySrcPaths = new Set(entries.map(e => e.srcPath))
// Create Soong modules and Make rules
let copyFiles = []
let namedModules = new Map<string, Module>()
let namedModules = new Map<string, SoongModule>()
for (let entry of entries) {
if (entry.isNamedDependency) {
// Named dependencies -> Soong blueprint
@ -161,172 +29,32 @@ async function generateBuild(
// Module name = file name
let ext = path.extname(entry.path)
let name = path.basename(entry.path, ext)
let moduleSrcPath = `proprietary/${entry.srcPath}`
// Skip if already done (e.g. other lib arch)
if (namedModules.has(name)) {
continue
}
// Type and info is based on file extension
let moduleSpecific: ModuleSpecific
if (ext == '.so') {
// Extract architecture from lib dir
let pathParts = entry.srcPath.split('/')
let libDir = pathParts.at(-2)
let curArch: string
if (libDir == 'lib') {
curArch = '32'
} else {
// Assume 64-bit native if not lib/lib64
curArch = '64'
}
let arch = curArch
// Check for the other arch
let otherLibDir = arch == '32' ? 'lib64' : 'lib'
let otherSrcPath = [
// Preceding parts
...pathParts.slice(0, -2),
// lib / lib64
otherLibDir,
// Trailing part (file name)
pathParts.at(-1),
].join('/')
if (entrySrcPaths.has(otherSrcPath)) {
// Both archs are present
arch = 'both'
}
// For single arch
let targetSrcs = {
srcs: [moduleSrcPath],
} as TargetSrcs
// For multi arch
let targetSrcs32 = (curArch == '32') ? targetSrcs : {
srcs: [`proprietary/${otherSrcPath}`],
} as TargetSrcs
let targetSrcs64 = (curArch == '64') ? targetSrcs : {
srcs: [`proprietary/${otherSrcPath}`],
} as TargetSrcs
moduleSpecific = {
_type: 'cc_prebuilt_library_shared',
strip: {
none: true,
},
target: {
...(arch == '32' && { android_arm: targetSrcs }),
...(arch == '64' && { android_arm64: targetSrcs }),
...(arch == 'both' && {
android_arm: targetSrcs32,
android_arm64: targetSrcs64,
}),
},
compile_multilib: arch,
check_elf_files: false,
prefer: true,
}
} else if (ext == '.apk') {
moduleSpecific = {
_type: 'android_app_import',
apk: moduleSrcPath,
...(entry.isPresigned && { presigned: true } || { certificate: 'platform' }),
...(entry.path.startsWith('priv-app/') && { privileged: true }),
dex_preopt: {
enabled: false,
},
}
} else if (ext == '.jar') {
moduleSpecific = {
_type: 'dex_import',
jars: [moduleSrcPath],
}
} else if (ext == '.xml') {
// Only etc/ XMLs are supported for now
let pathParts = entry.path.split('/')
if (pathParts[0] != 'etc') {
throw new Error(`XML file ${entry.srcPath} is not in etc/`)
}
moduleSpecific = {
_type: 'prebuilt_etc_xml',
src: moduleSrcPath,
filename_from_src: true,
sub_dir: pathParts.slice(1).join('/'),
}
} else {
throw new Error(`File ${entry.srcPath} has unknown extension ${ext}`)
}
let module = {
name: name,
owner: vendor,
...moduleSpecific,
// Partition flag
...(entry.partition == 'system_ext' && { system_ext_specific: true }),
...(entry.partition == 'product' && { product_specific: true }),
...(entry.partition == 'vendor' && { soc_specific: true }),
} as Module
let module = blobToSoongModule(name, ext, vendor, entry, entrySrcPaths)
namedModules.set(name, module)
} else {
// Other files -> Kati Makefile
// Simple PRODUCT_COPY_FILES line
let copyPart = entry.partition.toUpperCase()
copyFiles.push(`${proprietaryDir}/${entry.srcPath}:$(TARGET_COPY_OUT_${copyPart})/${entry.path}`)
copyFiles.push(blobToFileCopy(entry, proprietaryDir))
}
}
// Soong pass 2: serialize module objects
let serializedModules = []
for (let module of namedModules.values()) {
// Type prepended to Soong module props, so remove it from the object
let type = module._type;
delete module._type;
// Initial serialization pass. Node.js util.inspect happens to be identical to Soong format.
let serialized = util.inspect(module, {
depth: Infinity,
maxArrayLength: Infinity,
maxStringLength: Infinity,
})
// ' -> "
serialized = serialized.replaceAll("'", '"')
// 4-space indentation
serialized = serialized.replaceAll(' ', ' ')
// Prepend type
serialized = `${type} ${serialized}`
// Add trailing comma to last prop
let serialLines = serialized.split('\n')
serialLines[serialLines.length - 2] = serialLines.at(-2) + ','
serialized = serialLines.join('\n')
serializedModules.push(serialized)
}
let blueprint = `// Generated by adevtool, do not edit
soong_namespace {
}
${serializedModules.join('\n\n')}
`
let makefile = `# Generated by adevtool, do not edit
PRODUCT_SOONG_NAMESPACES += \\
${outDir}
PRODUCT_COPY_FILES += \\
${copyFiles.join(' \\\n ')}
`
// Serialize Soong blueprint
let blueprint = serializeBlueprint(namedModules.values())
fs.writeFile(`${outDir}/Android.bp`, blueprint)
// Serialize product makefile
let makefile = serializeProductMakefile({
namespaces: [outDir],
packages: Array.from(namedModules.keys()),
copyFiles: copyFiles,
})
fs.writeFile(`${outDir}/${device}-vendor.mk`, makefile)
}
@ -340,14 +68,15 @@ export default class Extract extends Command {
source: flags.string({char: 's', description: 'path to mounted factory images', required: true}),
}
static args = [{name: 'list'}]
static args = [{name: 'listPath'}]
async run() {
let {args: {list}, flags: {vendor, device, source}} = this.parse(Extract)
let {args: {listPath}, flags: {vendor, device, source}} = this.parse(Extract)
// Parse list
this.log(chalk.bold(chalk.greenBright('Parsing list')))
let entries = await parseList(list)
let list = await fs.readFile(listPath, {encoding: 'utf8'})
let entries = parseFileList(list)
// Prepare output directories
let outDir = `vendor/${vendor}/${device}`
@ -357,7 +86,7 @@ export default class Extract extends Command {
await fs.mkdir(proprietaryDir, {recursive: true})
// Copy blobs
//await copyBlobs(entries, source, proprietaryDir)
await copyBlobs(entries, source, proprietaryDir)
// Generate build files
this.log(chalk.bold(chalk.greenBright('Generating build files')))