generate-all: Follow generate rules in config

src/build/make.ts

@@ -40,7 +40,7 @@ export interface BoardMakefile {
   abOtaPartitions?: Array<string>
 
   boardInfo?: string
-  secontextResolutions?: SelinuxPartResolutions
+  sepolicyResolutions?: SelinuxPartResolutions
 }
 
 export interface DeviceMakefile {
@@ -164,8 +164,8 @@ TARGET_COPY_OUT_ODM_DLKM := odm_dlkm`)
     blocks.push(`TARGET_BOARD_INFO_FILE := ${mk.boardInfo}`)
   }
 
-  if (mk.secontextResolutions != undefined) {
+  if (mk.sepolicyResolutions != undefined) {
-    for (let [partition, {sepolicyDirs, missingContexts}] of mk.secontextResolutions.entries()) {
+    for (let [partition, {sepolicyDirs, missingContexts}] of mk.sepolicyResolutions.entries()) {
       let partVar = SEPOLICY_PARTITION_VARS[partition]
       if (sepolicyDirs.length > 0) {
         addContBlock(blocks, partVar, sepolicyDirs)

src/commands/generate-all.ts

@@ -1,27 +1,308 @@
 import { Command, flags } from '@oclif/command'
 import { promises as fs } from 'fs'
+import ora from 'ora'
 import path from 'path'
 import { flattenAllApexs } from '../blobs/apex'
 
-import { createVendorDirs, generateBuild, writeBuildFiles } from '../blobs/build'
+import { createVendorDirs, generateBuild, VendorDirectories, writeBuildFiles } from '../blobs/build'
 import { copyBlobs } from '../blobs/copy'
 import { BlobEntry } from '../blobs/entry'
 import { combinedPartPathToEntry, diffLists, listPart, serializeBlobList } from '../blobs/file_list'
 import { diffPartOverlays, parsePartOverlayApks, serializePartOverlays } from '../blobs/overlays'
 import { parsePresignedRecursive, updatePresignedBlobs } from '../blobs/presigned'
-import { diffPartitionProps, filterPartPropKeys, loadPartitionProps } from '../blobs/props'
+import { diffPartitionProps, loadPartitionProps, PartitionProps } from '../blobs/props'
 import { diffPartVintfManifests, loadPartVintfInfo, writePartVintfManifests } from '../blobs/vintf'
 import { findOverrideModules } from '../build/overrides'
 import { parseModuleInfo, removeSelfModules } from '../build/soong-info'
-import { loadDeviceConfig } from '../config/device'
+import { DeviceConfig, loadDeviceConfig } from '../config/device'
+import { filterKeys } from '../config/filters'
 import { parseSystemState, SystemState } from '../config/system-state'
 import { ANDROID_INFO, extractFactoryFirmware, generateAndroidInfo, writeFirmwareImages } from '../images/firmware'
-import { diffPartContexts, parseContextsRecursive, parsePartContexts, resolvePartContextDiffs, SelinuxContexts } from '../selinux/contexts'
+import { diffPartContexts, parseContextsRecursive, parsePartContexts, resolvePartContextDiffs, SelinuxContexts, SelinuxPartResolutions } from '../selinux/contexts'
 import { generateFileContexts } from '../selinux/labels'
-import { startActionSpinner, stopActionSpinner } from '../util/cli'
+import { withSpinner } from '../util/cli'
-import { readFile, withTempDir } from '../util/fs'
+import { readFile, TempState, withTempDir } from '../util/fs'
 import { ALL_SYS_PARTITIONS } from '../util/partitions'
 
+interface PropResults {
+  stockProps: PartitionProps
+  missingProps: PartitionProps
+
+  fingerprint: string
+  missingOtaParts: Array<string>
+}
+
+async function enumerateFiles(
+  spinner: ora.Ora,
+  namedEntries: Map<string, BlobEntry>,
+  customState: SystemState | null,
+  stockRoot: string,
+  customRoot: string,
+) {
+  for (let partition of ALL_SYS_PARTITIONS) {
+    let filesRef = await listPart(partition, stockRoot)
+    if (filesRef == null) continue
+    let filesNew = customState != null ? customState.partitionFiles[partition] :
+      await listPart(partition, customRoot)
+    if (filesNew == null) continue
+
+    let missingFiles = diffLists(filesNew, filesRef)
+
+    for (let combinedPartPath of missingFiles) {
+      let entry = combinedPartPathToEntry(partition, combinedPartPath)
+      namedEntries.set(combinedPartPath, entry)
+    }
+
+    spinner.text = partition
+  }
+}
+
+async function resolveOverrides(
+  config: DeviceConfig,
+  dirs: VendorDirectories,
+  namedEntries: Map<string, BlobEntry>,
+) {
+  let targetPrefix = `out/target/product/${config.device.name}/`
+  let targetPaths = Array.from(namedEntries.keys())
+    .map(cPartPath => `${targetPrefix}${cPartPath}`)
+
+  let moduleInfoPath = `${targetPrefix}module-info.json`
+  let modulesMap = parseModuleInfo(await readFile(moduleInfoPath))
+  removeSelfModules(modulesMap, dirs.proprietary)
+  let {modules: builtModules, builtPaths} = findOverrideModules(targetPaths, modulesMap)
+
+  // Remove new modules from entries
+  for (let path of builtPaths) {
+    namedEntries.delete(path.replace(targetPrefix, ''))
+  }
+
+  return builtModules
+}
+
+async function updatePresigned(
+  spinner: ora.Ora,
+  config: DeviceConfig,
+  entries: BlobEntry[],
+  aapt2Path: string,
+  stockRoot: string,
+) {
+  let presignedPkgs = await parsePresignedRecursive(config.platform.sepolicy_dirs)
+  await updatePresignedBlobs(aapt2Path, stockRoot, presignedPkgs, entries, entry => {
+    spinner.text = entry.srcPath
+  })
+}
+
+async function flattenApexs(
+  spinner: ora.Ora,
+  entries: BlobEntry[],
+  dirs: VendorDirectories,
+  tmp: TempState,
+  stockRoot: string,
+) {
+  let apex = await flattenAllApexs(entries, stockRoot, tmp, (progress) => {
+    spinner.text = progress
+  })
+
+  // Write context labels
+  let fileContexts = `${dirs.sepolicy}/file_contexts`
+  await fs.writeFile(fileContexts, generateFileContexts(apex.labels))
+
+  return apex.entries
+}
+
+async function extractProps(
+  config: DeviceConfig,
+  customState: SystemState | null,
+  stockRoot: string,
+  customRoot: string,
+) {
+  let stockProps = await loadPartitionProps(stockRoot)
+  let customProps = customState?.partitionProps ?? await loadPartitionProps(customRoot)
+
+  // Filters
+  for (let props of stockProps.values()) {
+    filterKeys(config.filters.props, props)
+  }
+  for (let props of customProps.values()) {
+    filterKeys(config.filters.props, props)
+  }
+
+  // Diff
+  let propChanges = diffPartitionProps(stockProps, customProps)
+  let missingProps = new Map(Array.from(propChanges.entries())
+    .map(([part, props]) => [part, props.removed]))
+
+  // Fingerprint for SafetyNet
+  let fingerprint = stockProps.get('system')!.get('ro.system.build.fingerprint')!
+
+  // A/B OTA partitions
+  let stockOtaParts = stockProps.get('product')!.get('ro.product.ab_ota_partitions')!.split(',')
+  let customOtaParts = new Set(customProps.get('product')!.get('ro.product.ab_ota_partitions')!.split(','))
+  // TODO: add proper filters
+  let missingOtaParts = stockOtaParts.filter(p => !customOtaParts.has(p) && p != 'vbmeta_vendor')
+
+  return {
+    stockProps,
+    missingProps,
+
+    fingerprint,
+    missingOtaParts,
+  } as PropResults
+}
+
+async function resolveSepolicyDirs(
+  config: DeviceConfig,
+  customState: SystemState | null,
+  dirs: VendorDirectories,
+  stockRoot: string,
+  customRoot: string,
+) {
+  // Built contexts
+  let stockContexts = await parsePartContexts(stockRoot)
+  let customContexts = customState?.partitionSecontexts ?? await parsePartContexts(customRoot)
+
+  // Contexts from AOSP
+  let sourceContexts: SelinuxContexts = new Map<string, string>()
+  for (let dir of config.platform.sepolicy_dirs) {
+    // TODO: support alternate ROM root
+    let contexts = await parseContextsRecursive(dir, '.')
+    for (let [ctx, source] of contexts.entries()) {
+      sourceContexts.set(ctx, source)
+    }
+  }
+
+  // Diff; reversed custom->stock order to get *missing* contexts
+  let ctxDiffs = diffPartContexts(customContexts, stockContexts)
+  let ctxResolutions = resolvePartContextDiffs(ctxDiffs, sourceContexts)
+
+  // Add APEX labels
+  if (ctxResolutions.has('vendor')) {
+    ctxResolutions.get('vendor')!.sepolicyDirs.push(dirs.sepolicy)
+  }
+
+  return ctxResolutions
+}
+
+async function extractOverlays(
+  spinner: ora.Ora,
+  customState: SystemState | null,
+  dirs: VendorDirectories,
+  aapt2Path: string,
+  stockRoot: string,
+  customRoot: string,
+) {
+  let stockOverlays = await parsePartOverlayApks(aapt2Path, stockRoot, path => {
+    spinner.text = path
+  })
+
+  let customOverlays = customState?.partitionOverlays ??
+    await parsePartOverlayApks(aapt2Path, customRoot, path => {
+      spinner.text = path
+    })
+
+  let missingOverlays = diffPartOverlays(stockOverlays, customOverlays)
+  return await serializePartOverlays(missingOverlays, dirs.overlays)
+}
+
+async function extractVintfManifests(
+  customState: SystemState | null,
+  dirs: VendorDirectories,
+  stockRoot: string,
+  customRoot: string,
+) {
+  let customVintf = customState?.partitionVintfInfo ?? await loadPartVintfInfo(customRoot)
+  let stockVintf = await loadPartVintfInfo(stockRoot)
+  let missingHals = diffPartVintfManifests(customVintf, stockVintf)
+
+  return await writePartVintfManifests(missingHals, dirs.vintf)
+}
+
+async function extractFirmware(
+  config: DeviceConfig,
+  dirs: VendorDirectories,
+  stockProps: PartitionProps,
+  factoryZip: string,
+) {
+  let fwImages = await extractFactoryFirmware(factoryZip)
+  let fwPaths = await writeFirmwareImages(fwImages, dirs.firmware)
+
+  // Generate android-info.txt from device and versions
+  let androidInfo = generateAndroidInfo(
+    config.device.name,
+    stockProps.get('vendor')!.get('ro.build.expect.bootloader')!,
+    stockProps.get('vendor')!.get('ro.build.expect.baseband')!,
+  )
+  await fs.writeFile(`${dirs.firmware}/${ANDROID_INFO}`, androidInfo)
+
+  return fwPaths
+}
+
+async function generateBuildFiles(
+  config: DeviceConfig,
+  dirs: VendorDirectories,
+  entries: BlobEntry[],
+  buildPkgs: string[],
+  propResults: PropResults | null,
+  fwPaths: string[] | null,
+  vintfManifestPaths: Map<string, string> | null,
+  sepolicyResolutions: SelinuxPartResolutions | null,
+  stockRoot: string,
+) {
+  let build = await generateBuild(entries, config.device.name, config.device.vendor, stockRoot, dirs)
+
+  // Add rules to build overridden modules and overlays, then re-sort
+  build.deviceMakefile!.packages!.push(...buildPkgs)
+  build.deviceMakefile!.packages!.sort((a, b) => a.localeCompare(b))
+
+  // Add device parts
+  build.deviceMakefile = {
+    props: propResults?.missingProps,
+    fingerprint: propResults?.fingerprint,
+    ...(vintfManifestPaths != null && { vintfManifestPaths: vintfManifestPaths }),
+    ...build.deviceMakefile,
+  }
+
+  // Add board parts
+  build.boardMakefile = {
+    ...(sepolicyResolutions != null && { sepolicyResolutions: sepolicyResolutions }),
+    ...(propResults != null && propResults.missingOtaParts.length > 0 &&
+      { abOtaPartitions: propResults.missingOtaParts }),
+    ...(fwPaths != null && { boardInfo: `${dirs.firmware}/${ANDROID_INFO}` }),
+  }
+
+  // Add firmware
+  if (fwPaths != null) {
+    build.modulesMakefile!.radioFiles = fwPaths.map(p => path.relative(dirs.out, p))
+  }
+
+  // Create device
+  if (config.generate.products) {
+    if (propResults == null) {
+      throw new Error('Product generation depends on properties')
+    }
+
+    let productProps = propResults.stockProps.get('product')!
+    let productName = productProps.get('ro.product.product.name')!
+
+    build.productMakefile = {
+      baseProductPath: config.platform.product_makefile,
+      name: productName,
+      model: productProps.get('ro.product.product.model')!,
+      brand: productProps.get('ro.product.product.brand')!,
+      manufacturer: productProps.get('ro.product.product.manufacturer')!,
+    }
+    build.productsMakefile = {
+      products: [productName],
+    }
+  }
+
+  // Dump list
+  let fileList = serializeBlobList(entries)
+  await fs.writeFile(`${dirs.out}/proprietary-files.txt`, fileList)
+
+  await writeBuildFiles(build, dirs)
+}
+
 export default class GenerateFull extends Command {
   static description = 'generate all vendor parts automatically'
 
@@ -56,207 +337,82 @@ export default class GenerateFull extends Command {
     let dirs = await createVendorDirs(config.device.vendor, config.device.name)
 
     // 1. Diff files
-    let spinner = startActionSpinner('Enumerating files')
+    await withSpinner('Enumerating files', (spinner) =>
-    for (let partition of ALL_SYS_PARTITIONS) {
+      enumerateFiles(spinner, namedEntries, customState, stockRoot, customRoot))
-      let filesRef = await listPart(partition, stockRoot)
-      if (filesRef == null) continue
-      let filesNew = customState != null ? customState.partitionFiles[partition] :
-        await listPart(partition, customRoot)
-      if (filesNew == null) continue
-
-      let missingFiles = diffLists(filesNew, filesRef)
-
-      for (let combinedPartPath of missingFiles) {
-        let entry = combinedPartPathToEntry(partition, combinedPartPath)
-        namedEntries.set(combinedPartPath, entry)
-      }
-
-      spinner.text = partition
-    }
-    stopActionSpinner(spinner)
 
     // 2. Overrides
-    spinner = startActionSpinner('Replacing blobs with buildable modules')
+    let buildPkgs: string[] = []
-    let targetPrefix = `out/target/product/${config.device.name}/`
+    if (config.generate.overrides) {
-    let targetPaths = Array.from(namedEntries.keys())
+      let builtModules = await withSpinner('Replacing blobs with buildable modules', () =>
-      .map(cPartPath => `${targetPrefix}${cPartPath}`)
+        resolveOverrides(config, dirs, namedEntries))
-
+      buildPkgs.push(...builtModules)
-    let moduleInfoPath = `${targetPrefix}module-info.json`
-    let modulesMap = parseModuleInfo(await readFile(moduleInfoPath))
-    removeSelfModules(modulesMap, dirs.proprietary)
-    let {modules: builtModules, builtPaths} = findOverrideModules(targetPaths, modulesMap)
-
-    // Remove new modules from entries
-    for (let path of builtPaths) {
-      namedEntries.delete(path.replace(targetPrefix, ''))
     }
-
     // After this point, we only need entry objects
     let entries = Array.from(namedEntries.values())
-    stopActionSpinner(spinner)
 
     // 3. Presigned
-    spinner = startActionSpinner('Finding presigned apps')
+    if (config.generate.presigned) {
-    let presignedPkgs = await parsePresignedRecursive(config.platform.sepolicy_dirs)
+      await withSpinner('Marking apps as presigned', (spinner) =>
-    await updatePresignedBlobs(aapt2Path, stockRoot, presignedPkgs, entries, entry => {
+        updatePresigned(spinner, config, entries, aapt2Path, stockRoot))
-      spinner.text = entry.srcPath
+    }
-    })
-    stopActionSpinner(spinner)
 
     // Create tmp dir in case we extract APEXs
     await withTempDir(async (tmp) => {
       // 4. Flatten APEX modules
       if (config.generate.flat_apex) {
-        spinner = startActionSpinner('Flattening APEX modules')
+        entries = await withSpinner('Flattening APEX modules', (spinner) =>
-        let apex = await flattenAllApexs(entries, stockRoot, tmp, (progress) => {
+          flattenApexs(spinner, entries, dirs, tmp, stockRoot))
-          spinner.text = progress
-        })
-        entries = apex.entries
-
-        // Write context labels
-        let fileContexts = `${dirs.sepolicy}/file_contexts`
-        await fs.writeFile(fileContexts, generateFileContexts(apex.labels))
-        stopActionSpinner(spinner)
       }
 
       // 5. Extract
       // Copy blobs (this has its own spinner)
-      if (!skipCopy) {
+      if (config.generate.files && !skipCopy) {
         await copyBlobs(entries, stockRoot, dirs.proprietary)
       }
 
       // 6. Props
-      spinner = startActionSpinner('Extracting properties')
+      let propResults: PropResults | null = null
-      let stockProps = await loadPartitionProps(stockRoot)
+      if (config.generate.props) {
-      let customProps = customState?.partitionProps ?? await loadPartitionProps(customRoot)
+        propResults = await withSpinner('Extracting properties', () =>
-      // Filter props
+          extractProps(config, customState, stockRoot, customRoot))
-      if (config.filters.props != null) {
-        filterPartPropKeys(stockProps, config.filters.props)
-        filterPartPropKeys(customProps, config.filters.props)
       }
-      // Diff
-      let propChanges = diffPartitionProps(stockProps, customProps)
-      let missingProps = new Map(Array.from(propChanges.entries())
-        .map(([part, props]) => [part, props.removed]))
-      // Fingerprint for SafetyNet
-      let fingerprint = stockProps.get('system')!.get('ro.system.build.fingerprint')!
-      // A/B OTA partitions
-      let stockOtaParts = stockProps.get('product')!.get('ro.product.ab_ota_partitions')!.split(',')
-      let customOtaParts = new Set(customProps.get('product')!.get('ro.product.ab_ota_partitions')!.split(','))
-      // TODO: add proper filters
-      let missingOtaParts = stockOtaParts.filter(p => !customOtaParts.has(p) && p != 'vbmeta_vendor')
-      stopActionSpinner(spinner)
 
       // 7. SELinux policies
-      spinner = startActionSpinner('Adding missing SELinux policies')
+      let sepolicyResolutions: SelinuxPartResolutions | null = null
-      // Built contexts
+      if (config.generate.sepolicy_dirs) {
-      let stockContexts = await parsePartContexts(stockRoot)
+        sepolicyResolutions = await withSpinner('Adding missing SELinux policies', () =>
-      let customContexts = customState?.partitionSecontexts ?? await parsePartContexts(customRoot)
+          resolveSepolicyDirs(config, customState, dirs, stockRoot, customRoot))
-      // Contexts from AOSP
-      let sourceContexts: SelinuxContexts = new Map<string, string>()
-      for (let dir of config.sepolicy_dirs) {
-        // TODO: support alternate ROM root
-        let contexts = await parseContextsRecursive(dir, '.')
-        for (let [ctx, source] of contexts.entries()) {
-          sourceContexts.set(ctx, source)
-        }
       }
-      // Diff; reversed custom->stock order to get *missing* contexts
-      let ctxDiffs = diffPartContexts(customContexts, stockContexts)
-      let ctxResolutions = resolvePartContextDiffs(ctxDiffs, sourceContexts)
-      // Add APEX labels
-      if (ctxResolutions.has('vendor')) {
-        ctxResolutions.get('vendor')!.sepolicyDirs.push(dirs.sepolicy)
-      }
-      stopActionSpinner(spinner)
 
       // 8. Overlays
-      spinner = startActionSpinner('Extracting overlays')
+      if (config.generate.overlays) {
-      let stockOverlays = await parsePartOverlayApks(aapt2Path, stockRoot, path => {
+        let overlayPkgs = await withSpinner('Extracting overlays', (spinner) =>
-        spinner.text = path
+          extractOverlays(spinner, customState, dirs, aapt2Path, stockRoot, customRoot))
-      })
+        buildPkgs.push(...overlayPkgs)
-      let customOverlays = customState?.partitionOverlays ??
+      }
-        await parsePartOverlayApks(aapt2Path, customRoot, path => {
-          spinner.text = path
-        })
-      let missingOverlays = diffPartOverlays(stockOverlays, customOverlays)
-      let overlayPkgs = await serializePartOverlays(missingOverlays, dirs.overlays)
-      stopActionSpinner(spinner)
 
       // 9. vintf manifests
-      spinner = startActionSpinner('Extracting vintf manifests')
+      let vintfManifestPaths: Map<string, string> | null = null
-      let customVintf = customState?.partitionVintfInfo ?? await loadPartVintfInfo(customRoot)
+      if (config.generate.vintf) {
-      let stockVintf = await loadPartVintfInfo(stockRoot)
+        vintfManifestPaths = await withSpinner('Extracting vintf manifests', () =>
-      let missingHals = diffPartVintfManifests(customVintf, stockVintf)
+          extractVintfManifests(customState, dirs, stockRoot, customRoot))
-      let vintfManifestPaths = await writePartVintfManifests(missingHals, dirs.vintf)
+      }
-      stopActionSpinner(spinner)
 
       // 10. Firmware
       let fwPaths: Array<string> | null = null
-      if (factoryZip != undefined) {
+      if (config.generate.factory_firmware && factoryZip != undefined) {
-        spinner = startActionSpinner('Extracting firmware')
+        if (propResults == null) {
-        let fwImages = await extractFactoryFirmware(factoryZip)
+          throw new Error('Factory firmware extraction depends on properties')
-        fwPaths = await writeFirmwareImages(fwImages, dirs.firmware)
+        }
 
-        // Generate android-info.txt from device and versions
+        fwPaths = await withSpinner('Extracting firmware', () =>
-        let androidInfo = generateAndroidInfo(
+          extractFirmware(config, dirs, propResults!.stockProps, factoryZip!))
-          config.device.name,
-          stockProps.get('vendor')!.get('ro.build.expect.bootloader')!,
-          stockProps.get('vendor')!.get('ro.build.expect.baseband')!,
-        )
-        await fs.writeFile(`${dirs.firmware}/${ANDROID_INFO}`, androidInfo)
-
-        stopActionSpinner(spinner)
       }
 
       // 11. Build files
-      spinner = startActionSpinner('Generating build files')
+      await withSpinner('Generating build files', () =>
-      let build = await generateBuild(entries, config.device.name, config.device.vendor, stockRoot, dirs)
+        generateBuildFiles(config, dirs, entries, buildPkgs, propResults, fwPaths,
-
+          vintfManifestPaths, sepolicyResolutions, stockRoot))
-      // Add rules to build overridden modules and overlays, then re-sort
-      build.deviceMakefile!.packages!.push(...builtModules, ...overlayPkgs)
-      build.deviceMakefile!.packages!.sort((a, b) => a.localeCompare(b))
-
-      // Add device parts
-      build.deviceMakefile = {
-        props: missingProps,
-        fingerprint: fingerprint,
-        vintfManifestPaths: vintfManifestPaths,
-        ...build.deviceMakefile,
-      }
-
-      // Add board parts
-      build.boardMakefile = {
-        secontextResolutions: ctxResolutions,
-        ...(missingOtaParts.length > 0 && { abOtaPartitions: missingOtaParts }),
-        ...(fwPaths != null && { boardInfo: `${dirs.firmware}/${ANDROID_INFO}` }),
-      }
-
-      // Add firmware
-      if (fwPaths != null) {
-        build.modulesMakefile!.radioFiles = fwPaths.map(p => path.relative(dirs.out, p))
-      }
-
-      // Create device
-      let productProps = stockProps.get('product')!
-      let productName = productProps.get('ro.product.product.name')!
-      build.productMakefile = {
-        baseProductPath: config.platform.product_makefile,
-        name: productName,
-        model: productProps.get('ro.product.product.model')!,
-        brand: productProps.get('ro.product.product.brand')!,
-        manufacturer: productProps.get('ro.product.product.manufacturer')!,
-      }
-      build.productsMakefile = {
-        products: [productName],
-      }
-
-      // Dump list
-      let fileList = serializeBlobList(entries)
-      await fs.writeFile(`${dirs.out}/proprietary-files.txt`, fileList)
-
-      await writeBuildFiles(build, dirs)
-      stopActionSpinner(spinner)
     })
   }
 }