diff --git a/toolkit/tools/imagecustomizer/docs/iso.md b/toolkit/tools/imagecustomizer/docs/iso.md index d15580d8a31..e8aae1ecfc9 100644 --- a/toolkit/tools/imagecustomizer/docs/iso.md +++ b/toolkit/tools/imagecustomizer/docs/iso.md @@ -2,26 +2,29 @@ ## Overview -Given a full disk image generated by the Azure Linux toolkit, the Azure Linux -Image Customizer can generate a LiveOS ISO image when the -`--output-image-format` is set to `iso`. - -The LiveOS ISO image is a bootable image that boots into the rootfs of the -input full disk image. This eliminates the requirement to install that image -to persistent storage before booting it. - -All customizations can still be made to the input full disk image rootfs as -usual using MIC, and will naturally carry over into the LiveOS ISO. This -includes customizations for kernel modules, dracut, and other early boot-time -actions. - -While converting the input full disk image into a LiveOS ISO involves copying +The Azure Linux Image Customizer can customize an input image and package the +output as a LiveOS iso image. The input image can be a full disk image +(vhd/vhdx/qcow2/raw) or another LiveOS iso image. The LiveOS iso output format +is specified by setting the `--output-image-format` command line parameter to +`iso`. + +The LiveOS iso image is a bootable image that boots into a rootfs image +included on the iso media without the need to have anything pre-installed +on the target machine. The rootfs is kept read-only and an overlay in memory +is created so that processes can successfully write state. + +Customizations are made to the input rootfs as usual using the Azure Linux +Image Customizer, and will naturally carry over into the LiveOS iso image. +This includes customizations for kernel modules, dracut, and other early +boot-time actions. + +While converting the input full disk image into a LiveOS iso involves copying almost all the artifacts unchanged - some artifacts are changed as follows: - `grub.cfg` is modified by updating the kernel command-line arguments as follows: - the root is updated to the LiveOS root file system image. - the LiveOS dracut parameters are appended. - - the MIC user-specified new parameters are appended. + - the user-specified new parameters are appended. - SELinux is disabled. - `/etc/fstab` is dropped from the rootfs as it typically conflicts with the overlay setup required by the LiveOS. @@ -31,16 +34,16 @@ almost all the artifacts unchanged - some artifacts are changed as follows: dracut configuration that got used before will be re-used again when we are re-generating the `initrd.img` and hence the original behavior is retained. -The current implementation for the LiveOS ISO does not support the following: +The current implementation for the LiveOS iso does not support the following: - persistent storage. - The machine loses all its state on reboot or shutdown. - dm-verity. - - The ISO image cannot run dm-verity for the LiveOS partitions. + - The iso image cannot run dm-verity for the LiveOS partitions. - disk layout. - There is always one disk generated when an `iso` output format is specified. - SELinux - - No SELinux configuration is supported for the generated ISO image. + - No SELinux configuration is supported for the generated iso image. - non-"Azure Linux toolkit" images - images generated by tools other than the Azure Linux toolkit may fail to be converted to a LiveOS iso. This is due to certain assumptions on the input @@ -80,11 +83,6 @@ iso: cloud-init-data/meta-data: /cloud-init-data/meta-data kernelCommandLine: ExtraCommandLine: "'ds=nocloud;s=file://run/initramfs/live/cloud-init-data'" -os: - users: - - name: test - password: testpassword - primaryGroup: sudo ``` #### Example 2 @@ -96,10 +94,6 @@ iso: kernelCommandLine: extraCommandLine: "'ds=nocloud;s=file://cloud-init-data'" os: - users: - - name: test - password: testpassword - primaryGroup: sudo additionalFiles: cloud-init-data/user-data: /cloud-init-data/user-data cloud-init-data/network-config: /cloud-init-data/network-config @@ -111,15 +105,20 @@ os: ### Full Disk Image The input full disk image must satisfy the following requirements in order for -MIC to be able to generate an iso image out of it: +the Azure Linux Image Cuztomizer to be able to generate an iso image out of it: - File layout (after all partitions have been mounted): - `/boot/grub2/grub.cfg` must exist and is the 'main' grub configuration (not a redirection grub configuration file for example). - The bootloader and the shim must exist under the `/boot` folder or any of its subdirectories. - For x64, `bootx64.efi` and `grubx64.efi` (or `grubx64-noprefix.efi`). - - For ARM64, `bootaa64.efi` and `grubaa64.efi` (or `grubx64-noprefix.efi`). + - For ARM64, `bootaa64.efi` and `grubaa64.efi` (or `grubaa64-noprefix.efi`). - All grub configurations and related files must be stored under the `/boot` folder. For example, grub.cfg cannot reference files outside that folder. If it does, those referenced files will not be copied to the iso and may cause grub to fail booting the desired operating system. + +### LiveOS ISO Image + +The input LiveOS iso image must be previously generated by the Azure Linux Image +Customizer. diff --git a/toolkit/tools/imagecustomizerapi/config.go b/toolkit/tools/imagecustomizerapi/config.go index 788c5be173c..a4caf67c0d7 100644 --- a/toolkit/tools/imagecustomizerapi/config.go +++ b/toolkit/tools/imagecustomizerapi/config.go @@ -8,16 +8,19 @@ import "fmt" type Config struct { Storage *Storage `yaml:"storage"` Iso *Iso `yaml:"iso"` - OS OS `yaml:"os"` - Scripts Scripts `yaml:"scripts"` + OS *OS `yaml:"os"` + Scripts *Scripts `yaml:"scripts"` } func (c *Config) IsValid() (err error) { + + hasStorage := false if c.Storage != nil { err = c.Storage.IsValid() if err != nil { return err } + hasStorage = true } if c.Iso != nil { @@ -27,19 +30,22 @@ func (c *Config) IsValid() (err error) { } } - err = c.OS.IsValid() - if err != nil { - return err + hasResetBootLoader := false + if c.OS != nil { + err = c.OS.IsValid() + if err != nil { + return err + } + hasResetBootLoader = c.OS.ResetBootLoaderType != ResetBootLoaderTypeDefault } - err = c.Scripts.IsValid() - if err != nil { - return err + if c.Scripts != nil { + err = c.Scripts.IsValid() + if err != nil { + return err + } } - hasStorage := c.Storage != nil - hasResetBootLoader := c.OS.ResetBootLoaderType != ResetBootLoaderTypeDefault - if hasStorage != hasResetBootLoader { return fmt.Errorf("os.resetBootLoaderType and storage must be specified together") } diff --git a/toolkit/tools/imagecustomizerapi/config_test.go b/toolkit/tools/imagecustomizerapi/config_test.go index d373b66b258..e55eda170c0 100644 --- a/toolkit/tools/imagecustomizerapi/config_test.go +++ b/toolkit/tools/imagecustomizerapi/config_test.go @@ -35,7 +35,7 @@ func TestConfigIsValid(t *testing.T) { }, }, }, - OS: OS{ + OS: &OS{ ResetBootLoaderType: "hard-reset", Hostname: "test", }, @@ -67,7 +67,7 @@ func TestConfigIsValidLegacy(t *testing.T) { }, }, }, - OS: OS{ + OS: &OS{ ResetBootLoaderType: "hard-reset", Hostname: "test", }, @@ -91,7 +91,7 @@ func TestConfigIsValidNoBootType(t *testing.T) { }, }}, }, - OS: OS{ + OS: &OS{ Hostname: "test", ResetBootLoaderType: "hard-reset", }, @@ -127,7 +127,7 @@ func TestConfigIsValidMissingBootLoaderReset(t *testing.T) { }, }, }, - OS: OS{ + OS: &OS{ Hostname: "test", }, } @@ -152,7 +152,7 @@ func TestConfigIsValidMultipleDisks(t *testing.T) { }, BootType: "legacy", }, - OS: OS{ + OS: &OS{ ResetBootLoaderType: "hard-reset", Hostname: "test", }, @@ -169,7 +169,7 @@ func TestConfigIsValidZeroDisks(t *testing.T) { BootType: BootTypeEfi, Disks: []Disk{}, }, - OS: OS{ + OS: &OS{ Hostname: "test", }, } @@ -181,7 +181,7 @@ func TestConfigIsValidZeroDisks(t *testing.T) { func TestConfigIsValidBadHostname(t *testing.T) { config := &Config{ - OS: OS{ + OS: &OS{ Hostname: "test_", }, } @@ -200,7 +200,7 @@ func TestConfigIsValidBadDisk(t *testing.T) { MaxSize: 0, }}, }, - OS: OS{ + OS: &OS{ Hostname: "test", }, } @@ -220,7 +220,7 @@ func TestConfigIsValidMissingEsp(t *testing.T) { }}, BootType: "efi", }, - OS: OS{ + OS: &OS{ ResetBootLoaderType: "hard-reset", Hostname: "test", }, @@ -242,7 +242,7 @@ func TestConfigIsValidMissingBiosBoot(t *testing.T) { }}, BootType: "legacy", }, - OS: OS{ + OS: &OS{ ResetBootLoaderType: "hard-reset", Hostname: "test", }, @@ -279,7 +279,7 @@ func TestConfigIsValidInvalidMountPoint(t *testing.T) { }, }, }, - OS: OS{ + OS: &OS{ ResetBootLoaderType: "hard-reset", Hostname: "test", }, @@ -316,7 +316,7 @@ func TestConfigIsValidKernelCLI(t *testing.T) { }, }, }, - OS: OS{ + OS: &OS{ ResetBootLoaderType: "hard-reset", Hostname: "test", KernelCommandLine: KernelCommandLine{ diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go b/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go index 1f9540d3cd0..c1f7afdce9e 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizeutils.go @@ -43,7 +43,7 @@ func doCustomizations(buildDir string, baseConfigPath string, config *imagecusto return err } - err = addRemoveAndUpdatePackages(buildDir, baseConfigPath, &config.OS, imageChroot, rpmsSources, + err = addRemoveAndUpdatePackages(buildDir, baseConfigPath, config.OS, imageChroot, rpmsSources, useBaseImageRpmRepos) if err != nil { return err @@ -112,9 +112,11 @@ func doCustomizations(buildDir string, baseConfigPath string, config *imagecusto } } - err = runScripts(baseConfigPath, config.Scripts.PostCustomization, imageChroot) - if err != nil { - return err + if config.Scripts != nil { + err = runScripts(baseConfigPath, config.Scripts.PostCustomization, imageChroot) + if err != nil { + return err + } } err = selinuxSetFiles(selinuxMode, imageChroot) @@ -127,9 +129,11 @@ func doCustomizations(buildDir string, baseConfigPath string, config *imagecusto return err } - err = runScripts(baseConfigPath, config.Scripts.FinalizeCustomization, imageChroot) - if err != nil { - return err + if config.Scripts != nil { + err = runScripts(baseConfigPath, config.Scripts.FinalizeCustomization, imageChroot) + if err != nil { + return err + } } return nil @@ -440,6 +444,9 @@ func handleBootLoader(baseConfigPath string, config *imagecustomizerapi.Config, case imagecustomizerapi.ResetBootLoaderTypeHard: logger.Log.Infof("Resetting bootloader config") + if config.Storage == nil { + return fmt.Errorf("failed to configure bootloader. Missing 'storage' configuration.") + } // Hard-reset the grub config. err := configureDiskBootLoader(imageConnection, config.Storage.FileSystems, config.Storage.BootType, config.OS.SELinux, config.OS.KernelCommandLine, currentSelinuxMode) diff --git a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go index 095567e240b..382725fde5f 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer.go @@ -43,6 +43,120 @@ var ( ToolVersion = "" ) +type ImageCustomizerParameters struct { + // build dirs + buildDir string + buildDirAbs string + + // input image + inputImageFile string + inputImageFormat string + inputIsIso bool + + // configurations + configPath string + config *imagecustomizerapi.Config + customizeOSPartitions bool + useBaseImageRpmRepos bool + rpmsSources []string + enableShrinkFilesystems bool + outputSplitPartitionsFormat string + + // intermediate writeable image + rawImageFile string + + // output image + outputImageFormat string + outputIsIso bool + qemuOutputImageFormat string + outputImageFile string + outputImageDir string + outputImageBase string +} + +func createImageCustomizerParameters(buildDir string, + inputImageFile string, + configPath string, config *imagecustomizerapi.Config, + useBaseImageRpmRepos bool, rpmsSources []string, enableShrinkFilesystems bool, outputSplitPartitionsFormat string, + outputImageFormat string, outputImageFile string) (*ImageCustomizerParameters, error) { + + ic := &ImageCustomizerParameters{} + + // working directories + ic.buildDir = buildDir + + buildDirAbs, err := filepath.Abs(buildDir) + if err != nil { + return nil, err + } + + ic.buildDirAbs = buildDirAbs + + // input image + ic.inputImageFile = inputImageFile + ic.inputImageFormat = strings.TrimLeft(filepath.Ext(inputImageFile), ".") + ic.inputIsIso = ic.inputImageFormat == ImageFormatIso + + // configuration + ic.configPath = configPath + ic.config = config + ic.customizeOSPartitions = (config.Storage != nil) || (config.OS != nil) || (config.Scripts != nil) + + ic.useBaseImageRpmRepos = useBaseImageRpmRepos + ic.rpmsSources = rpmsSources + + ic.enableShrinkFilesystems = enableShrinkFilesystems + ic.outputSplitPartitionsFormat = outputSplitPartitionsFormat + + // intermediate writeable image + ic.rawImageFile = filepath.Join(buildDirAbs, BaseImageName) + + // output image + ic.outputImageFormat = outputImageFormat + ic.outputIsIso = ic.outputImageFormat == ImageFormatIso + ic.outputImageFile = outputImageFile + ic.outputImageBase = strings.TrimSuffix(filepath.Base(outputImageFile), filepath.Ext(outputImageFile)) + ic.outputImageDir = filepath.Dir(outputImageFile) + + if ic.outputImageFormat != "" && !ic.outputIsIso { + ic.qemuOutputImageFormat, err = toQemuImageFormat(ic.outputImageFormat) + if err != nil { + return nil, err + } + } + + if ic.inputIsIso { + // When the input is an iso image, there's only one file system: the + // suqash file system and it has no empty space since it's a read-only + // file system. So, shrinking it does not make sense. + if ic.enableShrinkFilesystems { + return nil, fmt.Errorf("shrinking file systems is not supported when the input image is an iso image") + } + + // While splitting out the partition for an input iso can mean write + // the squash file system out to a raw image, we are choosing to + // not implement this until there is a need. + if ic.outputSplitPartitionsFormat != "" { + return nil, fmt.Errorf("extracting partitions is not supported when the input image is an iso image") + } + + // While re-creating a disk image from the iso is technically possible, + // we are choosing to not implement it until there is a need. + if !ic.outputIsIso { + return nil, fmt.Errorf("generating a non-iso image from an iso image is not supported") + } + + // While defining a storage configuration can work when the input image is + // an iso, there is no obvious point of moving content between partitions + // where all partitions get collapsed into the squashfs at the end. + if config.Storage != nil { + return nil, fmt.Errorf("cannot customize storage when the input is an iso") + } + } + + return ic, nil +} + func CustomizeImageWithConfigFile(buildDir string, configFile string, imageFile string, rpmsSources []string, outputImageFile string, outputImageFormat string, outputSplitPartitionsFormat string, useBaseImageRpmRepos bool, enableShrinkFilesystems bool, @@ -71,133 +185,226 @@ func CustomizeImageWithConfigFile(buildDir string, configFile string, imageFile return nil } +func cleanUp(ic *ImageCustomizerParameters) error { + err := file.RemoveFileIfExists(ic.rawImageFile) + if err != nil { + return err + } + + return nil +} + func CustomizeImage(buildDir string, baseConfigPath string, config *imagecustomizerapi.Config, imageFile string, rpmsSources []string, outputImageFile string, outputImageFormat string, outputSplitPartitionsFormat string, useBaseImageRpmRepos bool, enableShrinkFilesystems bool, ) error { - var err error - var qemuOutputImageFormat string - - outputImageBase := strings.TrimSuffix(filepath.Base(outputImageFile), filepath.Ext(outputImageFile)) - outputImageDir := filepath.Dir(outputImageFile) - - // Validate 'outputImageFormat' value if specified. - if outputImageFormat != "" && outputImageFormat != ImageFormatIso { - qemuOutputImageFormat, err = toQemuImageFormat(outputImageFormat) - if err != nil { - return err - } + err := validateConfig(baseConfigPath, config, rpmsSources, useBaseImageRpmRepos) + if err != nil { + return fmt.Errorf("invalid image config:\n%w", err) } - // Validate config. - err = validateConfig(baseConfigPath, config, rpmsSources, useBaseImageRpmRepos) + imageCustomizerParameters, err := createImageCustomizerParameters(buildDir, imageFile, + baseConfigPath, config, + useBaseImageRpmRepos, rpmsSources, enableShrinkFilesystems, outputSplitPartitionsFormat, + outputImageFormat, outputImageFile) if err != nil { - return fmt.Errorf("invalid image config:\n%w", err) + return fmt.Errorf("failed to create image customizer parameters object:\n%w", err) } + defer func() { + cleanupErr := cleanUp(imageCustomizerParameters) + if cleanupErr != nil { + if err != nil { + err = fmt.Errorf("%w:\nfailed to clean-up:\n%w", err, cleanupErr) + } else { + err = fmt.Errorf("failed to clean-up:\n%w", cleanupErr) + } + } + }() - // Normalize 'buildDir' path. - buildDirAbs, err := filepath.Abs(buildDir) + // ensure build and output folders are created up front + err = os.MkdirAll(imageCustomizerParameters.buildDirAbs, os.ModePerm) if err != nil { return err } - // Create 'buildDir' directory. - err = os.MkdirAll(buildDirAbs, os.ModePerm) + err = os.MkdirAll(imageCustomizerParameters.outputImageDir, os.ModePerm) if err != nil { return err } - // Convert image file to raw format, so that a kernel loop device can be used to make changes to the image. - rawImageFile := filepath.Join(buildDirAbs, BaseImageName) + inputIsoArtifacts, err := convertInputImageToWriteableFormat(imageCustomizerParameters) + if err != nil { + return fmt.Errorf("failed to convert input image to a raw image:\n%w", err) + } defer func() { - cleanupErr := file.RemoveFileIfExists(rawImageFile) - if cleanupErr != nil { - if err != nil { - err = fmt.Errorf("%w:\nfailed to clean-up (%s): %w", err, rawImageFile, cleanupErr) - } else { - err = fmt.Errorf("failed to clean-up (%s): %w", rawImageFile, cleanupErr) + if inputIsoArtifacts != nil { + cleanupErr := inputIsoArtifacts.cleanUp() + if cleanupErr != nil { + if err != nil { + err = fmt.Errorf("%w:\nfailed to clean-up iso builder state:\n%w", err, cleanupErr) + } else { + err = fmt.Errorf("failed to clean-up iso builder state:\n%w", cleanupErr) + } } } }() - logger.Log.Infof("Creating raw base image: %s", rawImageFile) - err = shell.ExecuteLiveWithErr(1, "qemu-img", "convert", "-O", "raw", imageFile, rawImageFile) + err = customizeOSContents(imageCustomizerParameters) + if err != nil { + return fmt.Errorf("failed to customize raw image:\n%w", err) + } + + err = convertWriteableFormatToOutputImage(imageCustomizerParameters, inputIsoArtifacts) if err != nil { - return fmt.Errorf("failed to convert image file to raw format:\n%w", err) + return fmt.Errorf("failed to convert customized raw image to output format:\n%w", err) + } + + logger.Log.Infof("Success!") + + return nil +} + +func convertInputImageToWriteableFormat(ic *ImageCustomizerParameters) (*LiveOSIsoBuilder, error) { + logger.Log.Infof("Converting input image to a writeable format") + + if ic.inputIsIso { + + inputIsoArtifacts, err := createIsoBuilderFromIsoImage(ic.buildDir, ic.buildDirAbs, ic.inputImageFile) + if err != nil { + return nil, fmt.Errorf("failed to load input iso artifacts:\n%w", err) + } + + // If the input is a LiveOS iso and there are OS customizations + // defined, we create a writeable disk image so that mic can modify + // it. If no OS customizations are defined, we can skip this step and + // just re-use the existing squashfs. + if ic.customizeOSPartitions { + err = inputIsoArtifacts.createWriteableImageFromSquashfs(ic.buildDir, ic.rawImageFile) + if err != nil { + return nil, fmt.Errorf("failed to create writeable image:\n%w", err) + } + } + return inputIsoArtifacts, nil + + } else { + logger.Log.Infof("Creating raw base image: %s", ic.rawImageFile) + err := shell.ExecuteLiveWithErr(1, "qemu-img", "convert", "-O", "raw", ic.inputImageFile, ic.rawImageFile) + if err != nil { + return nil, fmt.Errorf("failed to convert image file to raw format:\n%w", err) + } + } + + return nil, nil +} + +func customizeOSContents(ic *ImageCustomizerParameters) error { + // If there are OS customizations, then we proceed as usual. + // If there are no OS customizations, and the input is an iso, we just + // return because this function is mainly about OS customizations. + // This function also supports shrinking/exporting partitions. While + // we could support those functions for input isos, we are choosing to + // not support them until there is an actual need/a future time. + // We explicitly inform the user of the lack of support earlier during + // mic parameter validation (see createImageCustomizerParameters()). + if !ic.customizeOSPartitions && ic.inputIsIso { + return nil + } + + // The code beyond this point assumes the OS object is always present. To + // change the code to check before every usage whether the OS object is + // present or not will lead to a messy mix of if statements that do not + // serve the readibility of the code. A simpler solution is to instantiate + // a default imagecustomizerapi.OS object if the passed in one is absent. + // Then the code afterwards knows how to handle the default values + // correctly, and thus it eliminates the need for many if statements. + if ic.config.OS == nil { + ic.config.OS = &imagecustomizerapi.OS{} } // Check if the partition is using DM_verity_hash file system type. // The presence of this type indicates that dm-verity has been enabled on the base image. If dm-verity is not enabled, // the verity hash device should not be assigned this type. We do not support customization on verity enabled base // images at this time because such modifications would compromise the integrity and security mechanisms enforced by dm-verity. - err = checkDmVerityEnabled(rawImageFile) + err := checkDmVerityEnabled(ic.rawImageFile) if err != nil { return err } // Customize the partitions. - partitionsCustomized, rawImageFile, err := customizePartitions(buildDirAbs, baseConfigPath, config, rawImageFile) + partitionsCustomized, newRawImageFile, err := customizePartitions(ic.buildDirAbs, ic.configPath, ic.config, ic.rawImageFile) if err != nil { return err } + ic.rawImageFile = newRawImageFile // Customize the raw image file. - err = customizeImageHelper(buildDirAbs, baseConfigPath, config, rawImageFile, rpmsSources, useBaseImageRpmRepos, + err = customizeImageHelper(ic.buildDirAbs, ic.configPath, ic.config, ic.rawImageFile, ic.rpmsSources, ic.useBaseImageRpmRepos, partitionsCustomized) if err != nil { return err } // Shrink the filesystems. - if enableShrinkFilesystems { - err = shrinkFilesystemsHelper(rawImageFile) + if ic.enableShrinkFilesystems { + err = shrinkFilesystemsHelper(ic.rawImageFile) if err != nil { return fmt.Errorf("failed to shrink filesystems:\n%w", err) } } - if config.OS.Verity != nil { + if ic.config.OS.Verity != nil { // Customize image for dm-verity, setting up verity metadata and security features. - err = customizeVerityImageHelper(buildDirAbs, baseConfigPath, config, rawImageFile, rpmsSources, useBaseImageRpmRepos) + err = customizeVerityImageHelper(ic.buildDirAbs, ic.configPath, ic.config, ic.rawImageFile, ic.rpmsSources, ic.useBaseImageRpmRepos) if err != nil { return err } } // Check file systems for corruption. - err = checkFileSystems(rawImageFile) + err = checkFileSystems(ic.rawImageFile) if err != nil { return fmt.Errorf("failed to check filesystems:\n%w", err) } - // Create final output image file if requested. - switch outputImageFormat { - case ImageFormatVhd, ImageFormatVhdx, ImageFormatQCow2, ImageFormatRaw: - logger.Log.Infof("Writing: %s", outputImageFile) - - os.MkdirAll(outputImageDir, os.ModePerm) - err = shell.ExecuteLiveWithErr(1, "qemu-img", "convert", "-O", qemuOutputImageFormat, rawImageFile, outputImageFile) - if err != nil { - return fmt.Errorf("failed to convert image file to format: %s:\n%w", outputImageFormat, err) - } - case ImageFormatIso: - err = createLiveOSIsoImage(buildDir, baseConfigPath, config.Iso, rawImageFile, outputImageDir, outputImageBase) + // If outputSplitPartitionsFormat is specified, extract the partition files. + if ic.outputSplitPartitionsFormat != "" { + logger.Log.Infof("Extracting partition files") + err = extractPartitionsHelper(ic.rawImageFile, ic.outputImageDir, ic.outputImageBase, ic.outputSplitPartitionsFormat) if err != nil { return err } } - // If outputSplitPartitionsFormat is specified, extract the partition files. - if outputSplitPartitionsFormat != "" { - logger.Log.Infof("Extracting partition files") - err = extractPartitionsHelper(rawImageFile, outputImageDir, outputImageBase, outputSplitPartitionsFormat) + return nil +} + +func convertWriteableFormatToOutputImage(ic *ImageCustomizerParameters, inputIsoArtifacts *LiveOSIsoBuilder) error { + + logger.Log.Infof("Converting customized OS partitions into the final image") + + // Create final output image file if requested. + switch ic.outputImageFormat { + case ImageFormatVhd, ImageFormatVhdx, ImageFormatQCow2, ImageFormatRaw: + logger.Log.Infof("Writing: %s", ic.outputImageFile) + + err := shell.ExecuteLiveWithErr(1, "qemu-img", "convert", "-O", ic.qemuOutputImageFormat, ic.rawImageFile, ic.outputImageFile) if err != nil { - return err + return fmt.Errorf("failed to convert image file to format: %s:\n%w", ic.outputImageFormat, err) + } + case ImageFormatIso: + if ic.customizeOSPartitions { + err := createLiveOSIsoImage(ic.buildDir, ic.configPath, inputIsoArtifacts, ic.config.Iso, ic.rawImageFile, ic.outputImageDir, ic.outputImageBase) + if err != nil { + return fmt.Errorf("failed to create LiveOS iso image:\n%w", err) + } + } else { + err := inputIsoArtifacts.createImageFromUnchangedOS(ic.configPath, ic.config.Iso, ic.outputImageDir, ic.outputImageBase) + if err != nil { + return fmt.Errorf("failed to create LiveOS iso image:\n%w", err) + } } } - logger.Log.Infof("Success!") - return nil } @@ -229,12 +436,12 @@ func validateConfig(baseConfigPath string, config *imagecustomizerapi.Config, rp return err } - err = validateSystemConfig(baseConfigPath, &config.OS, rpmsSources, useBaseImageRpmRepos) + err = validateSystemConfig(baseConfigPath, config.OS, rpmsSources, useBaseImageRpmRepos) if err != nil { return err } - err = validateScripts(baseConfigPath, &config.Scripts) + err = validateScripts(baseConfigPath, config.Scripts) if err != nil { return err } @@ -278,6 +485,11 @@ func validateIsoConfig(baseConfigPath string, config *imagecustomizerapi.Iso) er func validateSystemConfig(baseConfigPath string, config *imagecustomizerapi.OS, rpmsSources []string, useBaseImageRpmRepos bool, ) error { + + if config == nil { + return nil + } + var err error err = validatePackageLists(baseConfigPath, config, rpmsSources, useBaseImageRpmRepos) @@ -294,6 +506,11 @@ func validateSystemConfig(baseConfigPath string, config *imagecustomizerapi.OS, } func validateScripts(baseConfigPath string, scripts *imagecustomizerapi.Scripts) error { + + if scripts == nil { + return nil + } + for i, script := range scripts.PostCustomization { err := validateScript(baseConfigPath, &script) if err != nil { @@ -337,6 +554,11 @@ func validateScript(baseConfigPath string, script *imagecustomizerapi.Script) er func validatePackageLists(baseConfigPath string, config *imagecustomizerapi.OS, rpmsSources []string, useBaseImageRpmRepos bool, ) error { + + if config == nil { + return nil + } + allPackagesRemove, err := collectPackagesList(baseConfigPath, config.Packages.RemoveLists, config.Packages.Remove) if err != nil { return err diff --git a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go index d91b5bca6ce..578207508ee 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imagecustomizer_test.go @@ -128,7 +128,7 @@ func connectToImage(buildDir string, imageFilePath string, mounts []mountPoint) func TestValidateConfigValidAdditionalFiles(t *testing.T) { err := validateConfig(testDir, &imagecustomizerapi.Config{ - OS: imagecustomizerapi.OS{ + OS: &imagecustomizerapi.OS{ AdditionalFiles: imagecustomizerapi.AdditionalFilesMap{ "files/a.txt": {{Path: "/a.txt"}}, }, @@ -138,7 +138,7 @@ func TestValidateConfigValidAdditionalFiles(t *testing.T) { func TestValidateConfigMissingAdditionalFiles(t *testing.T) { err := validateConfig(testDir, &imagecustomizerapi.Config{ - OS: imagecustomizerapi.OS{ + OS: &imagecustomizerapi.OS{ AdditionalFiles: imagecustomizerapi.AdditionalFilesMap{ "files/missing_a.txt": {{Path: "/a.txt"}}, }, @@ -148,7 +148,7 @@ func TestValidateConfigMissingAdditionalFiles(t *testing.T) { func TestValidateConfigdditionalFilesIsDir(t *testing.T) { err := validateConfig(testDir, &imagecustomizerapi.Config{ - OS: imagecustomizerapi.OS{ + OS: &imagecustomizerapi.OS{ AdditionalFiles: imagecustomizerapi.AdditionalFilesMap{ "files": {{Path: "/a.txt"}}, }, @@ -204,7 +204,7 @@ func TestCustomizeImageKernelCommandLineAdd(t *testing.T) { // Customize image. config := &imagecustomizerapi.Config{ - OS: imagecustomizerapi.OS{ + OS: &imagecustomizerapi.OS{ KernelCommandLine: imagecustomizerapi.KernelCommandLine{ ExtraCommandLine: "console=tty0 console=ttyS0", }, diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go index 608a8fe2b05..bb585112d8e 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go @@ -9,28 +9,36 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi" "github.com/microsoft/azurelinux/toolkit/tools/imagegen/configuration" + "github.com/microsoft/azurelinux/toolkit/tools/imagegen/diskutils" "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" + "github.com/microsoft/azurelinux/toolkit/tools/internal/safeloopback" + "github.com/microsoft/azurelinux/toolkit/tools/internal/safemount" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" "github.com/microsoft/azurelinux/toolkit/tools/pkg/isomakerlib" + "golang.org/x/sys/unix" ) const ( bootx64Binary = "bootx64.efi" grubx64Binary = "grubx64.efi" grubx64NoPrefixBinary = "grubx64-noprefix.efi" - grubCfg = "grub.cfg" + + grubCfg = "grub.cfg" searchCommandTemplate = "search --label %s --set root" rootValueTemplate = "live:LABEL=%s" - // The names initrd.img and vmlinuz are expected by isomaker. + isoBootDir = "boot" - isoInitrdPath = "/boot/initrd.img" + initrdImage = "initrd.img" + vmLinuzPrefix = "vmlinuz-" + isoInitrdPath = "/boot/" + initrdImage isoKernelPath = "/boot/vmlinuz" // kernel arguments template @@ -41,6 +49,10 @@ const ( dracutConfig = `add_dracutmodules+=" dmsquash-live " add_drivers+=" overlay " ` + // the total size of a collection of files is multiplied by the + // expansionSafetyFactor to estimate a disk size sufficient to hold those + // files. + expansionSafetyFactor = 1.5 ) type IsoWorkingDirs struct { @@ -65,12 +77,32 @@ type IsoArtifacts struct { vmlinuzPath string initrdImagePath string squashfsImagePath string - bootDirFiles map[string]string // local-build-path -> iso-media-path + additionalFiles map[string]string // local-build-path -> iso-media-path } type LiveOSIsoBuilder struct { workingDirs IsoWorkingDirs artifacts IsoArtifacts + cleanupDirs []string +} + +func (b *LiveOSIsoBuilder) addCleanupDir(dirName string) { + b.cleanupDirs = append(b.cleanupDirs, dirName) +} + +func (b *LiveOSIsoBuilder) cleanUp() error { + var err error + for i := len(b.cleanupDirs) - 1; i >= 0; i-- { + cleanupErr := os.RemoveAll(b.cleanupDirs[i]) + if cleanupErr != nil { + if err != nil { + err = fmt.Errorf("%w:\nfailed to remove (%s): %w", err, b.cleanupDirs[i], cleanupErr) + } else { + err = fmt.Errorf("failed to clean-up (%s): %w", b.cleanupDirs[i], cleanupErr) + } + } + } + return err } // populateWriteableRootfsDir @@ -190,16 +222,10 @@ func (b *LiveOSIsoBuilder) prepareRootfsForDracut(writeableRootfsDir string) err return fmt.Errorf("failed to delete fstab:\n%w", err) } - sourceConfigFile := filepath.Join(b.workingDirs.isoArtifactsDir, "20-live-cd.conf") - err = os.WriteFile(sourceConfigFile, []byte(dracutConfig), 0o644) - if err != nil { - return fmt.Errorf("failed to create %s:\n%w", sourceConfigFile, err) - } - targetConfigFile := filepath.Join(writeableRootfsDir, "/etc/dracut.conf.d/20-live-cd.conf") - err = file.Copy(sourceConfigFile, targetConfigFile) + err = os.WriteFile(targetConfigFile, []byte(dracutConfig), 0o644) if err != nil { - return fmt.Errorf("failed to copy dracut config at %s:\n%w", targetConfigFile, err) + return fmt.Errorf("failed to create %s:\n%w", targetConfigFile, err) } return nil @@ -301,10 +327,10 @@ func containsGrubNoPrefix(filePaths []string) bool { // b.artifacts.bootx64EfiPath // b.artifacts.grubx64EfiPath // b.artifacts.vmlinuzPath -// b.artifacts.bootDirFiles +// b.artifacts.additionalFiles func (b *LiveOSIsoBuilder) extractBootDirFiles(writeableRootfsDir string) error { - b.artifacts.bootDirFiles = make(map[string]string) + b.artifacts.additionalFiles = make(map[string]string) // the following files will be re-created - no need to copy them only to // have them overwritten. @@ -352,15 +378,21 @@ func (b *LiveOSIsoBuilder) extractBootDirFiles(writeableRootfsDir string) error targetPath := strings.Replace(sourcePath, writeableRootfsDir, b.workingDirs.isoArtifactsDir, -1) targetFileName := filepath.Base(targetPath) - copiedByIsoMaker := false + scheduleAdditionalFile := true switch targetFileName { case bootx64Binary: b.artifacts.bootx64EfiPath = targetPath - copiedByIsoMaker = true + // isomaker will extract this from initrd and copy it to include it + // in the iso media - so no need to schedule it as an additional + // file. + scheduleAdditionalFile = false case grubx64Binary, grubx64NoPrefixBinary: b.artifacts.grubx64EfiPath = targetPath - copiedByIsoMaker = true + // isomaker will extract this from initrd and copy it to include it + // in the iso media - so no need to schedule it as an additional + // file. + scheduleAdditionalFile = false case grubCfg: if usingGrubNoPrefix { // When using the grubx64-noprefix.efi, the 'prefix' grub @@ -381,11 +413,16 @@ func (b *LiveOSIsoBuilder) extractBootDirFiles(writeableRootfsDir string) error targetPath = filepath.Join(b.workingDirs.isoArtifactsDir, "EFI/BOOT", grubCfg) } b.artifacts.grubCfgPath = targetPath + // grub.cfg is passed as a parameter to isomaker. + scheduleAdditionalFile = false } - if strings.HasPrefix(targetFileName, "vmlinuz-") { + if strings.HasPrefix(targetFileName, vmLinuzPrefix) { targetPath = filepath.Join(filepath.Dir(targetPath), "vmlinuz") b.artifacts.vmlinuzPath = targetPath - copiedByIsoMaker = true + // isomaker will extract this from initrd and copy it to include it + // in the iso media - so no need to schedule it as an additional + // file. + scheduleAdditionalFile = false } err = file.NewFileCopyBuilder(sourcePath, targetPath). @@ -395,10 +432,8 @@ func (b *LiveOSIsoBuilder) extractBootDirFiles(writeableRootfsDir string) error return fmt.Errorf("failed to extract files from under the boot folder:\n%w", err) } - // If not copied by IsoMaker, add it to the list of files we will copy - // later. Otherwise, do not do anything and leave it to IsoMaker. - if !copiedByIsoMaker { - b.artifacts.bootDirFiles[targetPath] = strings.TrimPrefix(targetPath, b.workingDirs.isoArtifactsDir) + if scheduleAdditionalFile { + b.artifacts.additionalFiles[targetPath] = strings.TrimPrefix(targetPath, b.workingDirs.isoArtifactsDir) } } @@ -535,7 +570,7 @@ func (b *LiveOSIsoBuilder) createSquashfsImage(writeableRootfsDir string) error logger.Log.Debugf("Creating squashfs of %s", writeableRootfsDir) - squashfsImagePath := filepath.Join(b.workingDirs.isoArtifactsDir, "rootfs.img") + squashfsImagePath := filepath.Join(b.workingDirs.isoArtifactsDir, liveOSImage) exists, err := file.PathExists(squashfsImagePath) if err == nil && exists { @@ -552,6 +587,7 @@ func (b *LiveOSIsoBuilder) createSquashfsImage(writeableRootfsDir string) error } b.artifacts.squashfsImagePath = squashfsImagePath + return nil } @@ -602,7 +638,7 @@ func (b *LiveOSIsoBuilder) generateInitrdImage(rootfsSourceDir, artifactsSourceD } generatedInitrdPath := filepath.Join(rootfsSourceDir, initrdPathInChroot) - targetInitrdPath := filepath.Join(b.workingDirs.isoArtifactsDir, "initrd.img") + targetInitrdPath := filepath.Join(b.workingDirs.isoArtifactsDir, initrdImage) err = file.Copy(generatedInitrdPath, targetInitrdPath) if err != nil { return fmt.Errorf("failed to copy generated initrd:\n%w", err) @@ -720,7 +756,7 @@ func (b *LiveOSIsoBuilder) createIsoImage(additionalIsoFiles []safechroot.FileTo additionalIsoFiles = append(additionalIsoFiles, squashfsImageToCopy) // Add /boot/* files - for sourceFile, targetFile := range b.artifacts.bootDirFiles { + for sourceFile, targetFile := range b.artifacts.additionalFiles { fileToCopy := safechroot.FileToCopy{ Src: sourceFile, Dest: targetFile, @@ -816,6 +852,13 @@ func micIsoConfigToIsoMakerConfig(baseConfigPath string, isoConfig *imagecustomi // path to the folder where the mic configuration was loaded from. // This path will be used to construct absolute paths for file references // defined in the config. +// - 'inputIsoArtifacts' +// an optional LiveOSIsoBuilder that holds the state of the original input +// iso if one was provided. If present, this function will copy all files +// from the inputIsoArtifacts.artifacts.additionalFiles to the new iso +// if the destination is not already defined (for the new iso). +// This is used to carry over any files from a previously customized iso +// to the new one. // - 'isoConfig' // user provided configuration for the iso image. // - 'rawImageFile': @@ -829,7 +872,7 @@ func micIsoConfigToIsoMakerConfig(baseConfigPath string, isoConfig *imagecustomi // outputs: // // creates a LiveOS ISO image. -func createLiveOSIsoImage(buildDir, baseConfigPath string, isoConfig *imagecustomizerapi.Iso, rawImageFile, outputImageDir, outputImageBase string) (err error) { +func createLiveOSIsoImage(buildDir, baseConfigPath string, inputIsoArtifacts *LiveOSIsoBuilder, isoConfig *imagecustomizerapi.Iso, rawImageFile, outputImageDir, outputImageBase string) (err error) { additionalIsoFiles, extraCommandLine, err := micIsoConfigToIsoMakerConfig(baseConfigPath, isoConfig) if err != nil { @@ -869,6 +912,27 @@ func createLiveOSIsoImage(buildDir, baseConfigPath string, isoConfig *imagecusto return err } + // If we started from an input iso (not an input vhd(x)/qcow), then there + // might be additional files that are not defined in the current user + // configuration. Below, we loop through the files we have captured so far + // and append any file that was in the input iso and is not included + // already. This also ensures that no file from the input iso overwrites + // a newer version that has just been created. + if inputIsoArtifacts != nil { + for inputSourceFile, inputTargetFile := range inputIsoArtifacts.artifacts.additionalFiles { + found := false + for _, targetFile := range isoBuilder.artifacts.additionalFiles { + if inputTargetFile == targetFile { + found = true + break + } + } + if !found { + isoBuilder.artifacts.additionalFiles[inputSourceFile] = inputTargetFile + } + } + } + err = isoBuilder.createIsoImage(additionalIsoFiles, outputImageDir, outputImageBase) if err != nil { return err @@ -876,3 +940,450 @@ func createLiveOSIsoImage(buildDir, baseConfigPath string, isoConfig *imagecusto return nil } + +// extractIsoImageContents +// +// - given an iso image, this function extracts its contents into the specified +// folder. +// +// inputs: +// +// - 'buildDir': +// path build directory (can be shared with other tools). +// - 'isoImageFile' +// path to iso image file to extract its contents. +// - 'isoExpansionFolder' +// folder where the extracts contents will be copied to. +// +// outputs: +// +// - creates a local folder with the same structure and contents as the provided +// iso image. +func extractIsoImageContents(buildDir string, isoImageFile string, isoExpansionFolder string) (err error) { + + mountDir, err := os.MkdirTemp(buildDir, "tmp-iso-mount-") + if err != nil { + return fmt.Errorf("failed to create temporary mount folder for iso:\n%w", err) + } + defer os.RemoveAll(mountDir) + + isoImageLoopDevice, err := safeloopback.NewLoopback(isoImageFile) + if err != nil { + return fmt.Errorf("failed to create loop device for (%s):\n%w", isoImageFile, err) + } + defer isoImageLoopDevice.Close() + + isoImageMount, err := safemount.NewMount(isoImageLoopDevice.DevicePath(), mountDir, + "iso9660" /*fstype*/, unix.MS_RDONLY /*flags*/, "" /*data*/, false /*makeAndDelete*/) + if err != nil { + return err + } + defer isoImageMount.Close() + + err = os.MkdirAll(isoExpansionFolder, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create folder %s:\n%w", isoExpansionFolder, err) + } + + err = copyPartitionFiles(mountDir+"/.", isoExpansionFolder) + if err != nil { + return fmt.Errorf("failed to copy iso image contents to a writeable folder (%s):\n%w", isoExpansionFolder, err) + } + + err = isoImageMount.CleanClose() + if err != nil { + return err + } + + err = isoImageLoopDevice.CleanClose() + if err != nil { + return err + } + + return nil +} + +// createIsoBuilderFromIsoImage +// +// - given an iso image, this function extracts its contents, scans them, and +// constructs a LiveOSIsoBuilder object filling out as many of its fields as +// possible. +// +// inputs: +// +// - 'buildDir': +// path build directory (can be shared with other tools). +// - 'buildDirAbs' +// the absolute path of 'buildDir'. +// - 'isoImageFile' +// the source iso image file to extract/scan. +// +// outputs: +// +// - returns an instance of LiveOSIsoBuilder populated with all the paths of the +// extracted contents. +func createIsoBuilderFromIsoImage(buildDir string, buildDirAbs string, isoImageFile string) (isoBuilder *LiveOSIsoBuilder, err error) { + + isoBuildDir := filepath.Join(buildDir, "tmp") + isoBuilder = &LiveOSIsoBuilder{ + // + // buildDir (might be shared with other build tools) + // |--tmp (LiveOSIsoBuilder specific) + // |-- + // |--artifacts (extracted and generated artifacts) + // |--isomaker-tmp (used exclusively by isomaker) + // + workingDirs: IsoWorkingDirs{ + isoBuildDir: isoBuildDir, + isoArtifactsDir: filepath.Join(isoBuildDir, "artifacts"), + // IsoMaker needs its own folder to work in (it starts by deleting and re-creating it). + isomakerBuildDir: filepath.Join(isoBuildDir, "isomaker-tmp"), + }, + } + defer func() { + if err != nil { + cleanupErr := isoBuilder.cleanUp() + if cleanupErr != nil { + err = fmt.Errorf("%w:\nfailed to clean-up:\n%w", err, cleanupErr) + } + } + }() + + // create iso build folder + err = os.MkdirAll(isoBuildDir, os.ModePerm) + if err != nil { + return isoBuilder, fmt.Errorf("failed to create folder %s:\n%w", isoBuildDir, err) + } + isoBuilder.addCleanupDir(isoBuildDir) + + // extract iso contents + isoExpansionFolder, err := os.MkdirTemp(buildDirAbs, "expanded-input-iso-") + if err != nil { + return isoBuilder, fmt.Errorf("failed to create a temporary iso expansion folder for iso:\n%w", err) + } + isoBuilder.addCleanupDir(isoExpansionFolder) + + err = extractIsoImageContents(buildDir, isoImageFile, isoExpansionFolder) + if err != nil { + return isoBuilder, fmt.Errorf("failed to extract iso contents from input iso file:\n%w", err) + } + + isoFiles, err := file.EnumerateDirFiles(isoExpansionFolder) + if err != nil { + return isoBuilder, fmt.Errorf("failed to enumerate expanded iso files under %s:\n%w", isoExpansionFolder, err) + } + + isoBuilder.artifacts.additionalFiles = make(map[string]string) + + for _, isoFile := range isoFiles { + fileName := filepath.Base(isoFile) + + scheduleAdditionalFile := true + + switch fileName { + case bootx64Binary: + isoBuilder.artifacts.bootx64EfiPath = isoFile + // isomaker will extract this from initrd and copy it to include it + // in the iso media - so no need to schedule it as an additional + // file. + scheduleAdditionalFile = false + case grubx64Binary: + // Note that grubx64NoPrefixBinary is not expected to on an existing + // iso - and hence we do not look for it here. grubx64NoPrefixBinary + // may exist only on a vhdx/qcow when the grub-noprefix package is + // installed. When such images are converted to an iso, we rename + // the grub binary to its regular name (grubx64.efi). + isoBuilder.artifacts.grubx64EfiPath = isoFile + // isomaker will extract this from initrd and copy it to include it + // in the iso media - so no need to schedule it as an additional + // file. + scheduleAdditionalFile = false + case grubCfg: + isoBuilder.artifacts.grubCfgPath = isoFile + // grub.cfg is passed as a parameter to isomaker. + scheduleAdditionalFile = false + case liveOSImage: + isoBuilder.artifacts.squashfsImagePath = isoFile + // the squashfs image file is added to the additional file list + // by a different part of the code + scheduleAdditionalFile = false + case initrdImage: + isoBuilder.artifacts.initrdImagePath = isoFile + // initrd.img is passed as a parameter to isomaker. + scheduleAdditionalFile = false + } + if strings.HasPrefix(fileName, vmLinuzPrefix) { + isoBuilder.artifacts.vmlinuzPath = isoFile + // isomaker will extract this from initrd and copy it to include it + // in the iso media - so no need to schedule it as an additional + // file. + scheduleAdditionalFile = false + } + + if scheduleAdditionalFile { + isoBuilder.artifacts.additionalFiles[isoFile] = strings.TrimPrefix(isoFile, isoExpansionFolder) + } + } + + return isoBuilder, nil +} + +// createImageFromUnchangedOS +// +// - assuming the LiveOSIsoBuilder instance has all its artifacts populated, +// this function goes straight to updating grub and re-packaging the +// artifacts into an iso image. It does not re-create the initrd.img or +// the squashfs.img. This speeds-up customizing iso images when there are +// no customizations applicable to the OS (i.e. to the squashfs.img). +// +// inputs: +// +// - 'baseConfigPath': +// path to where the configuration is loaded from. This is used to resolve +// relative paths. +// - 'isoConfig' +// user provided configuration for the iso image. +// - 'outputImageDir': +// path to a folder where the generated iso will be placed. +// - 'outputImageBase': +// base name of the image to generate. The generated name will be on the +// form: {outputImageDir}/{outputImageBase}.iso +// +// outputs: +// +// - creates an iso image. +func (b *LiveOSIsoBuilder) createImageFromUnchangedOS(baseConfigPath string, isoConfig *imagecustomizerapi.Iso, outputImageDir string, outputImageBase string) error { + + logger.Log.Infof("Creating LiveOS iso image using unchanged OS partitions") + + additionalIsoFiles, extraCommandLine, err := micIsoConfigToIsoMakerConfig(baseConfigPath, isoConfig) + if err != nil { + return fmt.Errorf("failed to convert iso configuration to isomaker configuration format:\n%w", err) + } + + err = b.updateGrubCfg(b.artifacts.grubCfgPath, extraCommandLine) + if err != nil { + return fmt.Errorf("failed to update grub.cfg:\n%w", err) + } + + err = b.createIsoImage(additionalIsoFiles, outputImageDir, outputImageBase) + if err != nil { + return fmt.Errorf("failed to create iso image:\n%w", err) + } + + return nil +} + +// getSizeOnDiskInBytes +// +// - given a folder, it calculates the total size in bytes of its contents. +// +// inputs: +// +// - 'rootDir': +// root folder to calculate its size. +// +// outputs: +// +// - returns the size in bytes. +func getSizeOnDiskInBytes(rootDir string) (size uint64, err error) { + logger.Log.Debugf("Calculating total size for (%s)", rootDir) + + duStdout, _, err := shell.Execute("du", "-s", rootDir) + if err != nil { + return 0, fmt.Errorf("failed find the size of the specified folder using 'du' for (%s):\n%w", rootDir, err) + } + + // parse and get count and unit + diskSizeRegex := regexp.MustCompile(`^(\d+)\s+`) + matches := diskSizeRegex.FindStringSubmatch(duStdout) + if matches == nil || len(matches) < 2 { + return 0, fmt.Errorf("failed to parse 'du -s' output (%s).", duStdout) + } + + sizeInKbsString := matches[1] + sizeInKbs, err := strconv.ParseUint(sizeInKbsString, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse disk size (%d):\n%w", sizeInKbs, err) + } + + return sizeInKbs * diskutils.KiB, nil +} + +// getDiskSizeEstimateInMBs +// +// - given a folder, it calculates the size of a disk image that can hold +// all of its contents. +// - The amount of disk space a file occupies depends on the block size of the +// host file system. If many files are smaller than a block size, there will +// be a lot of waste. If files are very large, there will be very little +// waste. It is hard to predict how much disk space a set of a files will +// occupy without enumerating the sizes of all the files and knowing the +// target block size. In this function, we use an optimistic approach which +// calculates the required disk space by multiplying the total file size by +// a safety factor - i.e. safe that it will be able t hold all the contents. +// +// inputs: +// +// - 'rootDir': +// root folder to calculate its size. +// - 'safetyFactor': +// a multiplier used with the total number of bytes calculated. +// +// outputs: +// +// - returns the size in mega bytes. +func getDiskSizeEstimateInMBs(rootDir string, safetyFactor float64) (size uint64, err error) { + + sizeInBytes, err := getSizeOnDiskInBytes(rootDir) + if err != nil { + return 0, fmt.Errorf("failed to get folder size on disk while estimating total disk size:\n%w", err) + } + + sizeInMBs := sizeInBytes/diskutils.MiB + 1 + estimatedSizeInMBs := uint64(float64(sizeInMBs) * safetyFactor) + return estimatedSizeInMBs, nil +} + +// createWriteableImageFromSquashfs +// +// - given a squashfs image file, it creates a writeable image with two +// partitions, and copies the contents of the squashfs unto that writeable +// image. +// - the squashfs image file must be extracted from a previously created +// LiveOS iso and is specified by the LiveOSIsoBuilder.artifacts.squashfsImagePath. +// +// inputs: +// +// - 'buildDir': +// path build directory (can be shared with other tools). +// - 'rawImageFile': +// the name of the raw image to create and populate with the contents of +// the squashfs. +// +// outputs: +// +// - creates the specified writeable image. +func (b *LiveOSIsoBuilder) createWriteableImageFromSquashfs(buildDir, rawImageFile string) error { + + logger.Log.Infof("Creating writeable image from squashfs (%s)", b.artifacts.squashfsImagePath) + + // mount squash fs + squashMountDir, err := os.MkdirTemp(buildDir, "tmp-squashfs-mount-") + if err != nil { + return fmt.Errorf("failed to create temporary mount folder for squashfs:\n%w", err) + } + defer os.RemoveAll(squashMountDir) + + squashfsLoopDevice, err := safeloopback.NewLoopback(b.artifacts.squashfsImagePath) + if err != nil { + return fmt.Errorf("failed to create loop device for (%s):\n%w", b.artifacts.squashfsImagePath, err) + } + defer squashfsLoopDevice.Close() + + isoImageMount, err := safemount.NewMount(squashfsLoopDevice.DevicePath(), squashMountDir, + "squashfs" /*fstype*/, 0 /*flags*/, "" /*data*/, false /*makeAndDelete*/) + if err != nil { + return err + } + defer isoImageMount.Close() + + // estimate the new disk size + safeDiskSizeMB, err := getDiskSizeEstimateInMBs(squashMountDir, expansionSafetyFactor) + if err != nil { + return fmt.Errorf("failed to calculate the disk size of %s:\n%w", squashMountDir, err) + } + + logger.Log.Debugf("safeDiskSizeMB = %d", safeDiskSizeMB) + + // define a disk layout with a boot partition and a rootfs partition + maxDiskSizeMB := imagecustomizerapi.DiskSize(safeDiskSizeMB * diskutils.MiB) + var bootPartitionStart imagecustomizerapi.DiskSize + bootPartitionStart = imagecustomizerapi.DiskSize(1 * diskutils.MiB) + var bootPartitionEnd imagecustomizerapi.DiskSize + bootPartitionEnd = imagecustomizerapi.DiskSize(9 * diskutils.MiB) + + diskConfig := imagecustomizerapi.Disk{ + PartitionTableType: imagecustomizerapi.PartitionTableTypeGpt, + MaxSize: maxDiskSizeMB, + Partitions: []imagecustomizerapi.Partition{ + { + Id: "esp", + Start: bootPartitionStart, + End: &bootPartitionEnd, + Type: imagecustomizerapi.PartitionTypeESP, + }, + { + Id: "rootfs", + Start: bootPartitionEnd, + }, + }, + } + + fileSystemConfigs := []imagecustomizerapi.FileSystem{ + { + DeviceId: "esp", + Type: imagecustomizerapi.FileSystemTypeFat32, + MountPoint: &imagecustomizerapi.MountPoint{ + Path: "/boot/efi", + Options: "umask=0077", + }, + }, + { + DeviceId: "rootfs", + Type: imagecustomizerapi.FileSystemTypeExt4, + MountPoint: &imagecustomizerapi.MountPoint{ + Path: "/", + }, + }, + } + + // populate the newly created disk image with content from the squash fs + installOSFunc := func(imageChroot *safechroot.Chroot) error { + // At the point when this copy will be executed, both the boot and the + // root partitions will be mounted, and the files of /boot/efi will + // land on the the boot partition, while the rest will be on the rootfs + // partition. + err := copyPartitionFiles(squashMountDir+"/.", imageChroot.RootDir()) + if err != nil { + return fmt.Errorf("failed to copy squashfs contents to a writeable disk:\n%w", err) + } + + // In a vhd(s)/qcow->iso flow, grub.cfg can be modified by the use + // under the mic's os node, and then again later by the iso node. + // However, the second modification does not make it back into the + // squashfs since such customization is specific to the iso (live os + // configuration). Now, when we are creating a writeable image from an + // iso, we need to take the grub.cfg that in the iso (not the one in + // the squashfs), since it will have all previous user modifications + // and also the one actually used for booting. + sourceGrubCfgPath := b.artifacts.grubCfgPath + targetGrubCfgPath := filepath.Join(imageChroot.RootDir(), "boot/grub2/grub.cfg") + + err = file.Copy(sourceGrubCfgPath, targetGrubCfgPath) + if err != nil { + return fmt.Errorf("failed to copy the input iso grub.cfg to the writeable image:\n%w", err) + } + + return err + } + + // create the new raw disk image + writeableChrootDir := "writeable-raw-image" + err = createNewImage(rawImageFile, diskConfig, fileSystemConfigs, buildDir, writeableChrootDir, installOSFunc) + if err != nil { + return fmt.Errorf("failed to copy squashfs into new writeable image (%s):\n%w", rawImageFile, err) + } + + err = isoImageMount.CleanClose() + if err != nil { + return err + } + + err = squashfsLoopDevice.CleanClose() + if err != nil { + return err + } + + return nil +} diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder_test.go b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder_test.go index ff06a1ef701..4fdbd9776a3 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder_test.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder_test.go @@ -18,7 +18,7 @@ func TestCustomizeImageLiveCdIsoNoShimEfi(t *testing.T) { outImageFilePath := filepath.Join(buildDir, "image.iso") config := &imagecustomizerapi.Config{ - OS: imagecustomizerapi.OS{ + OS: &imagecustomizerapi.OS{ Packages: imagecustomizerapi.Packages{ Remove: []string{ "shim", @@ -40,7 +40,7 @@ func TestCustomizeImageLiveCdIsoNoGrubEfi(t *testing.T) { outImageFilePath := filepath.Join(buildDir, "image.iso") config := &imagecustomizerapi.Config{ - OS: imagecustomizerapi.OS{ + OS: &imagecustomizerapi.OS{ Packages: imagecustomizerapi.Packages{ Remove: []string{ "grub2-efi-binary",