~postmarketos/pmbootstrap

d9be146dc4d5e5ead3b1f43c40736814e07491f9 — Caleb Connolly 1 year, 4 months ago 2f9bdb7 buildenv-improvements
WIP: support using a buildroot as a disposable build environment

Adds support for using bindfs to bind-mount a directory from the host
into the chroot with UID/GID mapping to handle permissions issues

Sets up crossdirect in the buildroot for cross compiling

a PMBBUILD file can specify makedepends

TODO:
 * deal with bindfs being unavailable?
 * leverage envkernel logic to allow packaging build artifacts directly
   for easier testing
 * custom buildroots per-package
 * Set up sudo properly in the chroot
M pmb/chroot/apk.py => pmb/chroot/apk.py +12 -1
@@ 217,6 217,7 @@ def install(args, packages, suffix="native", build=True):
                  or needs to be updated, and it is inside pmaports. For the
                  special case that all packages are expected to be in Alpine's
                  repositories, set this to False for performance optimization.
    :param always: if False, only install packages that are not already
    """
    arch = pmb.parse.arch.from_chroot_suffix(args, suffix)



@@ 229,12 230,22 @@ def install(args, packages, suffix="native", build=True):
    check_min_version(args, suffix)
    pmb.chroot.init(args, suffix)

    installed_pkgs = installed(args, suffix)
    already_installed = all([pkg in installed_pkgs for pkg in packages])
    if not build and already_installed:
        return

    packages_with_depends = pmb.parse.depends.recurse(args, packages, suffix)
    to_add, to_del = packages_split_to_add_del(packages_with_depends)

    did_build = False
    if build:
        for package in to_add:
            install_build(args, package, arch)
            did_build |= bool(install_build(args, package, arch))
    
    if not did_build and already_installed:
        logging.debug(f"({suffix}) all packages already installed, skipping")
        return

    to_add_local = packages_get_locally_built_apks(args, to_add, arch)
    to_add_no_deps, _ = packages_split_to_add_del(packages)

M pmb/chroot/other.py => pmb/chroot/other.py +16 -0
@@ 73,3 73,19 @@ def copy_xauthority(args):
        pmb.helpers.run.root(args, ["rm", copy])
    pmb.helpers.run.root(args, ["cp", original, copy])
    pmb.chroot.root(args, ["chown", "pmos:pmos", "/home/pmos/.Xauthority"])

def sudo_nopasswd(args, suffix="native"):
    """
    Disable the password prompt for the wheel group in the chroot.
    """

    if "sudo" not in pmb.chroot.apk.installed(args, suffix):
        return


    if "rootfs" in suffix:
        logging.warning(f"!!!! SECURITY HOLE: Disabling sudo password for {suffix}")
    else:
        logging.info(f"({suffix}) disable sudo password prompt for wheel group")

    pmb.chroot.root(args, ["sed", "-i", "s/^# %wheel ALL=(ALL) ALL/%wheel ALL=(ALL) NOPASSWD: ALL/", "/etc/sudoers"], suffix)

M pmb/helpers/frontend.py => pmb/helpers/frontend.py +48 -3
@@ 10,6 10,7 @@ import pmb.aportgen
import pmb.build
import pmb.build.autodetect
import pmb.chroot
import pmb.chroot.apk
import pmb.chroot.initfs
import pmb.chroot.other
import pmb.ci


@@ 20,6 21,7 @@ import pmb.helpers.aportupgrade
import pmb.helpers.devices
import pmb.helpers.git
import pmb.helpers.lint
import pmb.helpers.mount
import pmb.helpers.logging
import pmb.helpers.pkgrel_bump
import pmb.helpers.pmaports


@@ 31,6 33,7 @@ import pmb.install
import pmb.install.blockdevice
import pmb.netboot
import pmb.parse
import pmb.parse.arch
import pmb.qemu
import pmb.sideload



@@ 150,6 153,8 @@ def netboot(args):
def chroot(args):
    # Suffix
    suffix = _parse_suffix(args)
    working_dir = "/"
    bindmount_targets = []
    if (args.user and suffix != "native" and
            not suffix.startswith("buildroot_")):
        raise RuntimeError("--user is only supported for native or"


@@ 157,6 162,15 @@ def chroot(args):
    if args.xauth and suffix != "native":
        raise RuntimeError("--xauth is only supported for native chroot.")

    if args.bind_mount:
        if suffix != "native" and not suffix.startswith("buildroot_"):
            raise RuntimeError("--bind-mount is only supported for native"
                               " or buildroot_* chroots.")
        if not all(os.path.isdir(src) for src in args.bind_mount):
            raise RuntimeError("bind-mount source must be a directory")

        args.user = True

    # apk: check minimum version, install packages
    pmb.chroot.apk.check_min_version(args, suffix)
    if args.add:


@@ 169,6 183,33 @@ def chroot(args):
        env["DISPLAY"] = os.environ.get("DISPLAY")
        env["XAUTHORITY"] = "/home/pmos/.Xauthority"

    if args.bind_mount:
        depends = []
        for src in args.bind_mount:
            src = os.path.realpath(src)
            name = os.path.basename(src)
            bindmount_targets.append(f"{args.work}/chroot_{suffix}/home/pmos/{name}")
            pmb.helpers.mount.bind(args, src, bindmount_targets[-1], bindfs=True)
            if os.path.isfile(f"{src}/PMBBUILD"):
                apk = pmb.parse.apkbuild(f"{src}/PMBBUILD", check_pkgname=False,
                                check_pkgver=False)
                logging.info("Adding makedepends to chroot: " + ",".join(apk["makedepends"]))
                depends += apk["makedepends"]

        arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
        pmb.build.init(args, suffix)
        pmb.build.other.configure_ccache(args, suffix)
        if suffix != "native":
            pmb.build.init_compiler(args, depends, "crossdirect", arch)
        pmb.chroot.mount_native_into_foreign(args, suffix)

        pmb.chroot.apk.install(args, depends, suffix)

        env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch,
                                pmb.config.chroot_path])

        working_dir = f"/home/pmos/{name}"

    # Install blockdevice
    if args.install_blockdev:
        size_boot = 128  # 128 MiB


@@ 181,12 222,16 @@ def chroot(args):
    if args.user:
        logging.info("(" + suffix + ") % su pmos -c '" +
                     " ".join(args.command) + "'")
        pmb.chroot.user(args, args.command, suffix, output=args.output,
                        env=env)
        try:
            pmb.chroot.user(args, args.command, suffix, output=args.output,
                            env=env, working_dir=working_dir)
        finally:
            for target in bindmount_targets:
                pmb.helpers.mount.umount_all(args, target)
    else:
        logging.info("(" + suffix + ") % " + " ".join(args.command))
        pmb.chroot.root(args, args.command, suffix, output=args.output,
                        env=env)
                        env=env, working_dir=working_dir)


def config(args):

M pmb/helpers/mount.py => pmb/helpers/mount.py +5 -2
@@ 20,7 20,7 @@ def ismount(folder):
    return False


def bind(args, source, destination, create_folders=True, umount=False):
def bind(args, source, destination, create_folders=True, umount=False, bindfs=False):
    """
    Mount --bind a folder and create necessary directory structure.
    :param umount: when destination is already a mount point, umount it first.


@@ 43,7 43,10 @@ def bind(args, source, destination, create_folders=True, umount=False):
                               path)

    # Actually mount the folder
    pmb.helpers.run.root(args, ["mount", "--bind", source, destination])
    if not bindfs:
        pmb.helpers.run.root(args, ["mount", "--bind", source, destination])
    else:
        pmb.helpers.run.root(args, ["bindfs", "--force-user=12345", "--force-group=12345", source, destination])

    # Verify that it has worked
    if not ismount(destination):

M pmb/parse/arguments.py => pmb/parse/arguments.py +10 -0
@@ 795,6 795,16 @@ def arguments():
                        help="Create a sparse image file and mount it as"
                              " /dev/install, just like during the"
                              " installation process.")
    chroot.add_argument("-m", "--bind-mount", nargs="*",
                        help="bind mount one or more"
                        " directories into the chroot.\n"
                        "If the directory contains a PMBBUILD file, it will"
                        " be parsed to automatically add the required"
                        " dependencies to the chroot.\n"
                        "All paths are mounted under /home/pmos/. The last"
                        " path specified is used as the working directory."
                        "Example: -m ~/pmos/unl0kr"
                        "Implies --user")
    for action in [build_init, chroot]:
        suffix = action.add_mutually_exclusive_group()
        if action == chroot: