diff --git a/src/build/make.ts b/src/build/make.ts index f40f728..4152a26 100644 --- a/src/build/make.ts +++ b/src/build/make.ts @@ -40,7 +40,7 @@ export interface BoardMakefile { abOtaPartitions?: Array boardInfo?: string - secontextResolutions?: SelinuxPartResolutions + sepolicyResolutions?: SelinuxPartResolutions } export interface DeviceMakefile { @@ -164,8 +164,8 @@ TARGET_COPY_OUT_ODM_DLKM := odm_dlkm`) blocks.push(`TARGET_BOARD_INFO_FILE := ${mk.boardInfo}`) } - if (mk.secontextResolutions != undefined) { - for (let [partition, {sepolicyDirs, missingContexts}] of mk.secontextResolutions.entries()) { + if (mk.sepolicyResolutions != undefined) { + for (let [partition, {sepolicyDirs, missingContexts}] of mk.sepolicyResolutions.entries()) { let partVar = SEPOLICY_PARTITION_VARS[partition] if (sepolicyDirs.length > 0) { addContBlock(blocks, partVar, sepolicyDirs) diff --git a/src/commands/generate-all.ts b/src/commands/generate-all.ts index 0473f08..a168b96 100644 --- a/src/commands/generate-all.ts +++ b/src/commands/generate-all.ts @@ -1,27 +1,308 @@ import { Command, flags } from '@oclif/command' import { promises as fs } from 'fs' +import ora from 'ora' import path from 'path' import { flattenAllApexs } from '../blobs/apex' -import { createVendorDirs, generateBuild, writeBuildFiles } from '../blobs/build' +import { createVendorDirs, generateBuild, VendorDirectories, writeBuildFiles } from '../blobs/build' import { copyBlobs } from '../blobs/copy' import { BlobEntry } from '../blobs/entry' import { combinedPartPathToEntry, diffLists, listPart, serializeBlobList } from '../blobs/file_list' import { diffPartOverlays, parsePartOverlayApks, serializePartOverlays } from '../blobs/overlays' import { parsePresignedRecursive, updatePresignedBlobs } from '../blobs/presigned' -import { diffPartitionProps, filterPartPropKeys, loadPartitionProps } from '../blobs/props' +import { diffPartitionProps, loadPartitionProps, PartitionProps } from '../blobs/props' import { diffPartVintfManifests, loadPartVintfInfo, writePartVintfManifests } from '../blobs/vintf' import { findOverrideModules } from '../build/overrides' import { parseModuleInfo, removeSelfModules } from '../build/soong-info' -import { loadDeviceConfig } from '../config/device' +import { DeviceConfig, loadDeviceConfig } from '../config/device' +import { filterKeys } from '../config/filters' import { parseSystemState, SystemState } from '../config/system-state' import { ANDROID_INFO, extractFactoryFirmware, generateAndroidInfo, writeFirmwareImages } from '../images/firmware' -import { diffPartContexts, parseContextsRecursive, parsePartContexts, resolvePartContextDiffs, SelinuxContexts } from '../selinux/contexts' +import { diffPartContexts, parseContextsRecursive, parsePartContexts, resolvePartContextDiffs, SelinuxContexts, SelinuxPartResolutions } from '../selinux/contexts' import { generateFileContexts } from '../selinux/labels' -import { startActionSpinner, stopActionSpinner } from '../util/cli' -import { readFile, withTempDir } from '../util/fs' +import { withSpinner } from '../util/cli' +import { readFile, TempState, withTempDir } from '../util/fs' import { ALL_SYS_PARTITIONS } from '../util/partitions' +interface PropResults { + stockProps: PartitionProps + missingProps: PartitionProps + + fingerprint: string + missingOtaParts: Array +} + +async function enumerateFiles( + spinner: ora.Ora, + namedEntries: Map, + customState: SystemState | null, + stockRoot: string, + customRoot: string, +) { + for (let partition of ALL_SYS_PARTITIONS) { + let filesRef = await listPart(partition, stockRoot) + if (filesRef == null) continue + let filesNew = customState != null ? customState.partitionFiles[partition] : + await listPart(partition, customRoot) + if (filesNew == null) continue + + let missingFiles = diffLists(filesNew, filesRef) + + for (let combinedPartPath of missingFiles) { + let entry = combinedPartPathToEntry(partition, combinedPartPath) + namedEntries.set(combinedPartPath, entry) + } + + spinner.text = partition + } +} + +async function resolveOverrides( + config: DeviceConfig, + dirs: VendorDirectories, + namedEntries: Map, +) { + let targetPrefix = `out/target/product/${config.device.name}/` + let targetPaths = Array.from(namedEntries.keys()) + .map(cPartPath => `${targetPrefix}${cPartPath}`) + + let moduleInfoPath = `${targetPrefix}module-info.json` + let modulesMap = parseModuleInfo(await readFile(moduleInfoPath)) + removeSelfModules(modulesMap, dirs.proprietary) + let {modules: builtModules, builtPaths} = findOverrideModules(targetPaths, modulesMap) + + // Remove new modules from entries + for (let path of builtPaths) { + namedEntries.delete(path.replace(targetPrefix, '')) + } + + return builtModules +} + +async function updatePresigned( + spinner: ora.Ora, + config: DeviceConfig, + entries: BlobEntry[], + aapt2Path: string, + stockRoot: string, +) { + let presignedPkgs = await parsePresignedRecursive(config.platform.sepolicy_dirs) + await updatePresignedBlobs(aapt2Path, stockRoot, presignedPkgs, entries, entry => { + spinner.text = entry.srcPath + }) +} + +async function flattenApexs( + spinner: ora.Ora, + entries: BlobEntry[], + dirs: VendorDirectories, + tmp: TempState, + stockRoot: string, +) { + let apex = await flattenAllApexs(entries, stockRoot, tmp, (progress) => { + spinner.text = progress + }) + + // Write context labels + let fileContexts = `${dirs.sepolicy}/file_contexts` + await fs.writeFile(fileContexts, generateFileContexts(apex.labels)) + + return apex.entries +} + +async function extractProps( + config: DeviceConfig, + customState: SystemState | null, + stockRoot: string, + customRoot: string, +) { + let stockProps = await loadPartitionProps(stockRoot) + let customProps = customState?.partitionProps ?? await loadPartitionProps(customRoot) + + // Filters + for (let props of stockProps.values()) { + filterKeys(config.filters.props, props) + } + for (let props of customProps.values()) { + filterKeys(config.filters.props, props) + } + + // Diff + let propChanges = diffPartitionProps(stockProps, customProps) + let missingProps = new Map(Array.from(propChanges.entries()) + .map(([part, props]) => [part, props.removed])) + + // Fingerprint for SafetyNet + let fingerprint = stockProps.get('system')!.get('ro.system.build.fingerprint')! + + // A/B OTA partitions + let stockOtaParts = stockProps.get('product')!.get('ro.product.ab_ota_partitions')!.split(',') + let customOtaParts = new Set(customProps.get('product')!.get('ro.product.ab_ota_partitions')!.split(',')) + // TODO: add proper filters + let missingOtaParts = stockOtaParts.filter(p => !customOtaParts.has(p) && p != 'vbmeta_vendor') + + return { + stockProps, + missingProps, + + fingerprint, + missingOtaParts, + } as PropResults +} + +async function resolveSepolicyDirs( + config: DeviceConfig, + customState: SystemState | null, + dirs: VendorDirectories, + stockRoot: string, + customRoot: string, +) { + // Built contexts + let stockContexts = await parsePartContexts(stockRoot) + let customContexts = customState?.partitionSecontexts ?? await parsePartContexts(customRoot) + + // Contexts from AOSP + let sourceContexts: SelinuxContexts = new Map() + for (let dir of config.platform.sepolicy_dirs) { + // TODO: support alternate ROM root + let contexts = await parseContextsRecursive(dir, '.') + for (let [ctx, source] of contexts.entries()) { + sourceContexts.set(ctx, source) + } + } + + // Diff; reversed custom->stock order to get *missing* contexts + let ctxDiffs = diffPartContexts(customContexts, stockContexts) + let ctxResolutions = resolvePartContextDiffs(ctxDiffs, sourceContexts) + + // Add APEX labels + if (ctxResolutions.has('vendor')) { + ctxResolutions.get('vendor')!.sepolicyDirs.push(dirs.sepolicy) + } + + return ctxResolutions +} + +async function extractOverlays( + spinner: ora.Ora, + customState: SystemState | null, + dirs: VendorDirectories, + aapt2Path: string, + stockRoot: string, + customRoot: string, +) { + let stockOverlays = await parsePartOverlayApks(aapt2Path, stockRoot, path => { + spinner.text = path + }) + + let customOverlays = customState?.partitionOverlays ?? + await parsePartOverlayApks(aapt2Path, customRoot, path => { + spinner.text = path + }) + + let missingOverlays = diffPartOverlays(stockOverlays, customOverlays) + return await serializePartOverlays(missingOverlays, dirs.overlays) +} + +async function extractVintfManifests( + customState: SystemState | null, + dirs: VendorDirectories, + stockRoot: string, + customRoot: string, +) { + let customVintf = customState?.partitionVintfInfo ?? await loadPartVintfInfo(customRoot) + let stockVintf = await loadPartVintfInfo(stockRoot) + let missingHals = diffPartVintfManifests(customVintf, stockVintf) + + return await writePartVintfManifests(missingHals, dirs.vintf) +} + +async function extractFirmware( + config: DeviceConfig, + dirs: VendorDirectories, + stockProps: PartitionProps, + factoryZip: string, +) { + let fwImages = await extractFactoryFirmware(factoryZip) + let fwPaths = await writeFirmwareImages(fwImages, dirs.firmware) + + // Generate android-info.txt from device and versions + let androidInfo = generateAndroidInfo( + config.device.name, + stockProps.get('vendor')!.get('ro.build.expect.bootloader')!, + stockProps.get('vendor')!.get('ro.build.expect.baseband')!, + ) + await fs.writeFile(`${dirs.firmware}/${ANDROID_INFO}`, androidInfo) + + return fwPaths +} + +async function generateBuildFiles( + config: DeviceConfig, + dirs: VendorDirectories, + entries: BlobEntry[], + buildPkgs: string[], + propResults: PropResults | null, + fwPaths: string[] | null, + vintfManifestPaths: Map | null, + sepolicyResolutions: SelinuxPartResolutions | null, + stockRoot: string, +) { + let build = await generateBuild(entries, config.device.name, config.device.vendor, stockRoot, dirs) + + // Add rules to build overridden modules and overlays, then re-sort + build.deviceMakefile!.packages!.push(...buildPkgs) + build.deviceMakefile!.packages!.sort((a, b) => a.localeCompare(b)) + + // Add device parts + build.deviceMakefile = { + props: propResults?.missingProps, + fingerprint: propResults?.fingerprint, + ...(vintfManifestPaths != null && { vintfManifestPaths: vintfManifestPaths }), + ...build.deviceMakefile, + } + + // Add board parts + build.boardMakefile = { + ...(sepolicyResolutions != null && { sepolicyResolutions: sepolicyResolutions }), + ...(propResults != null && propResults.missingOtaParts.length > 0 && + { abOtaPartitions: propResults.missingOtaParts }), + ...(fwPaths != null && { boardInfo: `${dirs.firmware}/${ANDROID_INFO}` }), + } + + // Add firmware + if (fwPaths != null) { + build.modulesMakefile!.radioFiles = fwPaths.map(p => path.relative(dirs.out, p)) + } + + // Create device + if (config.generate.products) { + if (propResults == null) { + throw new Error('Product generation depends on properties') + } + + let productProps = propResults.stockProps.get('product')! + let productName = productProps.get('ro.product.product.name')! + + build.productMakefile = { + baseProductPath: config.platform.product_makefile, + name: productName, + model: productProps.get('ro.product.product.model')!, + brand: productProps.get('ro.product.product.brand')!, + manufacturer: productProps.get('ro.product.product.manufacturer')!, + } + build.productsMakefile = { + products: [productName], + } + } + + // Dump list + let fileList = serializeBlobList(entries) + await fs.writeFile(`${dirs.out}/proprietary-files.txt`, fileList) + + await writeBuildFiles(build, dirs) +} + export default class GenerateFull extends Command { static description = 'generate all vendor parts automatically' @@ -56,207 +337,82 @@ export default class GenerateFull extends Command { let dirs = await createVendorDirs(config.device.vendor, config.device.name) // 1. Diff files - let spinner = startActionSpinner('Enumerating files') - for (let partition of ALL_SYS_PARTITIONS) { - let filesRef = await listPart(partition, stockRoot) - if (filesRef == null) continue - let filesNew = customState != null ? customState.partitionFiles[partition] : - await listPart(partition, customRoot) - if (filesNew == null) continue - - let missingFiles = diffLists(filesNew, filesRef) - - for (let combinedPartPath of missingFiles) { - let entry = combinedPartPathToEntry(partition, combinedPartPath) - namedEntries.set(combinedPartPath, entry) - } - - spinner.text = partition - } - stopActionSpinner(spinner) + await withSpinner('Enumerating files', (spinner) => + enumerateFiles(spinner, namedEntries, customState, stockRoot, customRoot)) // 2. Overrides - spinner = startActionSpinner('Replacing blobs with buildable modules') - let targetPrefix = `out/target/product/${config.device.name}/` - let targetPaths = Array.from(namedEntries.keys()) - .map(cPartPath => `${targetPrefix}${cPartPath}`) - - let moduleInfoPath = `${targetPrefix}module-info.json` - let modulesMap = parseModuleInfo(await readFile(moduleInfoPath)) - removeSelfModules(modulesMap, dirs.proprietary) - let {modules: builtModules, builtPaths} = findOverrideModules(targetPaths, modulesMap) - - // Remove new modules from entries - for (let path of builtPaths) { - namedEntries.delete(path.replace(targetPrefix, '')) + let buildPkgs: string[] = [] + if (config.generate.overrides) { + let builtModules = await withSpinner('Replacing blobs with buildable modules', () => + resolveOverrides(config, dirs, namedEntries)) + buildPkgs.push(...builtModules) } - // After this point, we only need entry objects let entries = Array.from(namedEntries.values()) - stopActionSpinner(spinner) // 3. Presigned - spinner = startActionSpinner('Finding presigned apps') - let presignedPkgs = await parsePresignedRecursive(config.platform.sepolicy_dirs) - await updatePresignedBlobs(aapt2Path, stockRoot, presignedPkgs, entries, entry => { - spinner.text = entry.srcPath - }) - stopActionSpinner(spinner) + if (config.generate.presigned) { + await withSpinner('Marking apps as presigned', (spinner) => + updatePresigned(spinner, config, entries, aapt2Path, stockRoot)) + } // Create tmp dir in case we extract APEXs await withTempDir(async (tmp) => { // 4. Flatten APEX modules if (config.generate.flat_apex) { - spinner = startActionSpinner('Flattening APEX modules') - let apex = await flattenAllApexs(entries, stockRoot, tmp, (progress) => { - spinner.text = progress - }) - entries = apex.entries - - // Write context labels - let fileContexts = `${dirs.sepolicy}/file_contexts` - await fs.writeFile(fileContexts, generateFileContexts(apex.labels)) - stopActionSpinner(spinner) + entries = await withSpinner('Flattening APEX modules', (spinner) => + flattenApexs(spinner, entries, dirs, tmp, stockRoot)) } // 5. Extract // Copy blobs (this has its own spinner) - if (!skipCopy) { + if (config.generate.files && !skipCopy) { await copyBlobs(entries, stockRoot, dirs.proprietary) } // 6. Props - spinner = startActionSpinner('Extracting properties') - let stockProps = await loadPartitionProps(stockRoot) - let customProps = customState?.partitionProps ?? await loadPartitionProps(customRoot) - // Filter props - if (config.filters.props != null) { - filterPartPropKeys(stockProps, config.filters.props) - filterPartPropKeys(customProps, config.filters.props) + let propResults: PropResults | null = null + if (config.generate.props) { + propResults = await withSpinner('Extracting properties', () => + extractProps(config, customState, stockRoot, customRoot)) } - // Diff - let propChanges = diffPartitionProps(stockProps, customProps) - let missingProps = new Map(Array.from(propChanges.entries()) - .map(([part, props]) => [part, props.removed])) - // Fingerprint for SafetyNet - let fingerprint = stockProps.get('system')!.get('ro.system.build.fingerprint')! - // A/B OTA partitions - let stockOtaParts = stockProps.get('product')!.get('ro.product.ab_ota_partitions')!.split(',') - let customOtaParts = new Set(customProps.get('product')!.get('ro.product.ab_ota_partitions')!.split(',')) - // TODO: add proper filters - let missingOtaParts = stockOtaParts.filter(p => !customOtaParts.has(p) && p != 'vbmeta_vendor') - stopActionSpinner(spinner) // 7. SELinux policies - spinner = startActionSpinner('Adding missing SELinux policies') - // Built contexts - let stockContexts = await parsePartContexts(stockRoot) - let customContexts = customState?.partitionSecontexts ?? await parsePartContexts(customRoot) - // Contexts from AOSP - let sourceContexts: SelinuxContexts = new Map() - for (let dir of config.sepolicy_dirs) { - // TODO: support alternate ROM root - let contexts = await parseContextsRecursive(dir, '.') - for (let [ctx, source] of contexts.entries()) { - sourceContexts.set(ctx, source) - } + let sepolicyResolutions: SelinuxPartResolutions | null = null + if (config.generate.sepolicy_dirs) { + sepolicyResolutions = await withSpinner('Adding missing SELinux policies', () => + resolveSepolicyDirs(config, customState, dirs, stockRoot, customRoot)) } - // Diff; reversed custom->stock order to get *missing* contexts - let ctxDiffs = diffPartContexts(customContexts, stockContexts) - let ctxResolutions = resolvePartContextDiffs(ctxDiffs, sourceContexts) - // Add APEX labels - if (ctxResolutions.has('vendor')) { - ctxResolutions.get('vendor')!.sepolicyDirs.push(dirs.sepolicy) - } - stopActionSpinner(spinner) // 8. Overlays - spinner = startActionSpinner('Extracting overlays') - let stockOverlays = await parsePartOverlayApks(aapt2Path, stockRoot, path => { - spinner.text = path - }) - let customOverlays = customState?.partitionOverlays ?? - await parsePartOverlayApks(aapt2Path, customRoot, path => { - spinner.text = path - }) - let missingOverlays = diffPartOverlays(stockOverlays, customOverlays) - let overlayPkgs = await serializePartOverlays(missingOverlays, dirs.overlays) - stopActionSpinner(spinner) + if (config.generate.overlays) { + let overlayPkgs = await withSpinner('Extracting overlays', (spinner) => + extractOverlays(spinner, customState, dirs, aapt2Path, stockRoot, customRoot)) + buildPkgs.push(...overlayPkgs) + } // 9. vintf manifests - spinner = startActionSpinner('Extracting vintf manifests') - let customVintf = customState?.partitionVintfInfo ?? await loadPartVintfInfo(customRoot) - let stockVintf = await loadPartVintfInfo(stockRoot) - let missingHals = diffPartVintfManifests(customVintf, stockVintf) - let vintfManifestPaths = await writePartVintfManifests(missingHals, dirs.vintf) - stopActionSpinner(spinner) + let vintfManifestPaths: Map | null = null + if (config.generate.vintf) { + vintfManifestPaths = await withSpinner('Extracting vintf manifests', () => + extractVintfManifests(customState, dirs, stockRoot, customRoot)) + } // 10. Firmware let fwPaths: Array | null = null - if (factoryZip != undefined) { - spinner = startActionSpinner('Extracting firmware') - let fwImages = await extractFactoryFirmware(factoryZip) - fwPaths = await writeFirmwareImages(fwImages, dirs.firmware) + if (config.generate.factory_firmware && factoryZip != undefined) { + if (propResults == null) { + throw new Error('Factory firmware extraction depends on properties') + } - // Generate android-info.txt from device and versions - let androidInfo = generateAndroidInfo( - config.device.name, - stockProps.get('vendor')!.get('ro.build.expect.bootloader')!, - stockProps.get('vendor')!.get('ro.build.expect.baseband')!, - ) - await fs.writeFile(`${dirs.firmware}/${ANDROID_INFO}`, androidInfo) - - stopActionSpinner(spinner) + fwPaths = await withSpinner('Extracting firmware', () => + extractFirmware(config, dirs, propResults!.stockProps, factoryZip!)) } // 11. Build files - spinner = startActionSpinner('Generating build files') - let build = await generateBuild(entries, config.device.name, config.device.vendor, stockRoot, dirs) - - // Add rules to build overridden modules and overlays, then re-sort - build.deviceMakefile!.packages!.push(...builtModules, ...overlayPkgs) - build.deviceMakefile!.packages!.sort((a, b) => a.localeCompare(b)) - - // Add device parts - build.deviceMakefile = { - props: missingProps, - fingerprint: fingerprint, - vintfManifestPaths: vintfManifestPaths, - ...build.deviceMakefile, - } - - // Add board parts - build.boardMakefile = { - secontextResolutions: ctxResolutions, - ...(missingOtaParts.length > 0 && { abOtaPartitions: missingOtaParts }), - ...(fwPaths != null && { boardInfo: `${dirs.firmware}/${ANDROID_INFO}` }), - } - - // Add firmware - if (fwPaths != null) { - build.modulesMakefile!.radioFiles = fwPaths.map(p => path.relative(dirs.out, p)) - } - - // Create device - let productProps = stockProps.get('product')! - let productName = productProps.get('ro.product.product.name')! - build.productMakefile = { - baseProductPath: config.platform.product_makefile, - name: productName, - model: productProps.get('ro.product.product.model')!, - brand: productProps.get('ro.product.product.brand')!, - manufacturer: productProps.get('ro.product.product.manufacturer')!, - } - build.productsMakefile = { - products: [productName], - } - - // Dump list - let fileList = serializeBlobList(entries) - await fs.writeFile(`${dirs.out}/proprietary-files.txt`, fileList) - - await writeBuildFiles(build, dirs) - stopActionSpinner(spinner) + await withSpinner('Generating build files', () => + generateBuildFiles(config, dirs, entries, buildPkgs, propResults, fwPaths, + vintfManifestPaths, sepolicyResolutions, stockRoot)) }) } }