From 0fb7adb0184ec6c9c2d7f372ccaead84d6df5627 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Thu, 25 May 2023 14:49:10 -0400 Subject: [PATCH 1/4] hack: add build-image for abstracting the hard parts of image building Add a python-based script `build-image` that takes on the complexities of how we build our images, including applying common tags and "fully-qualified image names". The script can be called from a makefile but doesn't require it. When used directly you can generate multiple image variants in one pass. Example: ``` ./hack/build-image -k server -p default -p nightly -a amd64 -a arm64 ``` Will produce four images, in short: 1. server with default packages on amd64 2. server with default packages on arm64 3. server with nightly packages on amd64 4. server with nightly packages on arm64 (all using the default distro, currently fedora) Signed-off-by: John Mulligan --- hack/build-image | 593 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 593 insertions(+) create mode 100755 hack/build-image diff --git a/hack/build-image b/hack/build-image new file mode 100755 index 00000000..11892ad4 --- /dev/null +++ b/hack/build-image @@ -0,0 +1,593 @@ +#!/usr/bin/python3 +""" +build-image - A script for building a matrix of samba container images. + +In order to cleanly implement both logic and maintain backwards compatibility +with image naming schemes already in the wild the build-image script +can be used to create "Fully Qualified Image Names" that combine the +image kind (samba-server, client, etc) and a tag that indicates all +the unique properties of the image. This includes the package source, +the base os, and the architecture. + +In addition to building the images, one can push images, list images, +and list build status files (aka buildfiles). + +Usage: + # build an image + ./hack/build-image --kind server --distro-base fedora --arch amd64 + + # print out the FQIN + ./hack/build-image --kind samba-server --distro-base fedora \\ + --arch amd64 --print + + # print out the FQIN and additional tags + ./hack/build-image --kind samba-server --distro-base fedora \\ + --arch amd64 --print-tags + + # print out the FQIN and additional tags for multiple images, with + # and without a repository base + ./hack/build-image --kind samba-server \\ + --distro-base fedora \\ + --distro-base centos \\ + --distro-base opensuse \\ + --arch amd64 \\ + --repo-base quay.io/foobar --without-repo-bases --print-tags + +""" + +import argparse +import logging +import os +import pathlib +import platform +import shlex +import shutil +import subprocess +import sys + + +logger = logging.getLogger("build-image") + +# IMAGE_KINDS - map aliases/names to canonical names for the kinds +# of images we can build +IMG_SERVER = "samba-server" +IMG_AD_SERVER = "samba-ad-server" +IMG_CLIENT = "samba-client" +IMG_TOOLBOX = "samba-toolbox" +IMAGE_KINDS = { + # short names + "server": IMG_SERVER, + "ad-server": IMG_AD_SERVER, + "client": IMG_CLIENT, + "toolbox": IMG_TOOLBOX, + # canonical names + "samba-server": IMG_SERVER, + "samba-ad-server": IMG_AD_SERVER, + "samba-client": IMG_CLIENT, + "samba-toolbox": IMG_TOOLBOX, +} + +# ARCHITECTURES - map supported arch names/alias to canonical names +AMD64 = "amd64" +ARM64 = "arm64" +ARCHITECTURES = { + # alternate names + "x86_64": AMD64, + "aarch64": ARM64, + # canonical names + "amd64": AMD64, + "arm64": ARM64, +} + +# DISTROS - list of supported distro bases +FEDORA = "fedora" +CENTOS = "centos" +OPENSUSE = "opensuse" +DISTROS = [ + FEDORA, + CENTOS, + OPENSUSE, +] + +# PACKAGE_SOURCES - list of known package sources +DEFAULT = "default" +NIGHTLY = "nightly" +PACKAGE_SOURCES = [DEFAULT, NIGHTLY] + +# SOURCE_DIRS - image source paths +# (paths are relative to common image source dir) +SOURCE_DIRS = { + IMG_SERVER: "server", + IMG_AD_SERVER: "ad-server", + IMG_CLIENT: "client", + IMG_TOOLBOX: "toolbox", +} + +DEFAULT_PKG_SOURCES = [DEFAULT] +DEFAULT_DISTRO_BASES = [FEDORA] +LATEST = "latest" +QUAL_NONE = "unqualified" +QUAL_DISTRO = "distro-qualified" + + +_DISCOVERED_CONTAINER_ENGINES = [] + + +def check_kind(kind): + """Return the canonical name for the image kind or raise a ValueError.""" + try: + return IMAGE_KINDS[kind] + except KeyError: + raise ValueError(f"invalid kind: {kind}") + + +def check_arch(arch): + """Return the canonical name for the arch or raise a ValueError.""" + try: + return ARCHITECTURES[arch] + except KeyError: + raise ValueError(f"invalid arch: {arch}") + + +def check_distro(distro): + """Return the canonical name for a distro base or raise a ValueError.""" + if distro in DISTROS: + return distro + raise ValueError(f"invalid distro: {distro}") + + +def check_pkg_source(source): + """Return the canonical name for a package source or raise a ValueError.""" + if source in PACKAGE_SOURCES: + return source + raise ValueError(f"invalid package source: {source}") + + +def check_repo_base_for(value): + """Return a tuple with a (distro_base, repo_base) pair or raise a + ValueError. + """ + if "=" not in value: + raise ValueError("expected '=' in value") + db, rb = value.split("=", 1) + db = check_distro(db) + return (db, rb) + + +def _cmd_to_str(cmd): + """Format a command for logging.""" + return " ".join(shlex.quote(arg) for arg in cmd) + + +def run(cli, cmd, capture_output=False, check=False): + """Execute a command. Wraps subprocess.run.""" + if cli.dry_run and not capture_output: + logger.info("Would run: %s", _cmd_to_str(cmd)) + return subprocess.CompletedProcess(cmd, 0) + logger.info("Running: %s", _cmd_to_str(cmd)) + return subprocess.run(cmd, capture_output=capture_output, check=check) + + +def container_engine(cli): + """Return the path to a container engine. If the container engine is not + yet known, discover it and cache the result. + """ + eng = cli.container_engine + if eng: + logger.info("Using specified container engine: %s", eng) + return eng + if _DISCOVERED_CONTAINER_ENGINES: + return _DISCOVERED_CONTAINER_ENGINES[0] + podman = shutil.which("podman") + if podman: + _DISCOVERED_CONTAINER_ENGINES.append(podman) + docker = shutil.which("docker") + if docker: + _DISCOVERED_CONTAINER_ENGINES.append(docker) + return _DISCOVERED_CONTAINER_ENGINES[0] + + +def container_build(cli, target): + """Construct and execute a command to build the target container image.""" + args = [container_engine(cli), "build"] + if target.pkg_source == NIGHTLY: + args.append("--build-arg=INSTALL_PACKAGES_FROM=samba-nightly") + # docker doesn't currently support alt. architectures + if "docker" in args[0]: + if target.arch != host_arch(): + raise RuntimeError("Docker does not support --arch") + else: + args.append(f"--arch={target.arch}") + if cli.extra_build_arg: + args.extend(cli.extra_build_arg) + for tname in target.all_names(baseless=cli.without_repo_bases): + args.append("-t") + args.append(tname) + args.append("-f") + args.append(target_containerfile(target)) + args.append(kind_source_dir(target.name)) + args = [str(a) for a in args] + run(cli, args, check=True) + + +def container_push(cli, push_name): + """Construct and execute a command to push a container image.""" + args = [container_engine(cli), "push", push_name] + run(cli, args, check=True) + + +def container_id(cli, target): + """Construct and run a command to fetch a hexidecimal id for a container + image. + """ + args = [ + container_engine(cli), + "inspect", + "-f", + "{{.Id}}", + target.image_name(), + ] + res = run(cli, args, capture_output=True, check=True) + return res.stdout.decode("utf8").strip() + + +def kind_source_dir(kind): + """Return the path to a kind's source directory.""" + return pathlib.Path("images") / SOURCE_DIRS[check_kind(kind)] + + +def target_containerfile(target): + """Return the path to a containerfile given an image target.""" + return str(kind_source_dir(target.name) / f"Containerfile.{target.distro}") + + +def host_arch(): + """Return the name of the host's native architecture.""" + return check_arch(platform.machine().lower()) + + +def default_arches(): + """Return a list of the default architectures to use for building.""" + return [host_arch()] + + +class RepoConfig: + def __init__(self, default_repo_base, distro_repo=None): + self.default = default_repo_base + self.distro_map = dict(distro_repo or []) + + def find_base(self, distro): + return self.distro_map.get(distro, self.default) + + +class TargetImage: + def __init__( + self, name, pkg_source, distro, arch, extra_tag="", *, repo_base="" + ): + self.name = name + self.pkg_source = pkg_source + self.distro = distro + self.arch = arch + self.extra_tag = extra_tag + self.repo_base = repo_base + self.additional_tags = [] + + def tag_name(self): + tag_parts = [self.pkg_source, self.distro, self.arch] + if self.extra_tag: + tag_parts.append(self.extra_tag) + tag = "-".join(tag_parts) + return tag + + def image_name(self, *, tag=None, repo_base=None): + if not tag: + tag = self.tag_name() + image_name = f"{self.name}:{tag}" + repo_base = repo_base if repo_base is not None else self.repo_base + if repo_base: + repo_base = repo_base.rstrip("/") + image_name = f"{repo_base}/{image_name}" + return image_name + + def flat_name(self): + return f"{self.name}.{self.tag_name()}" + + def __str__(self): + return self.image_name() + + def all_names(self, baseless=False): + yield self.image_name() + for tag, _ in self.additional_tags: + yield self.image_name(tag=tag) + if self.repo_base and baseless: + yield self.image_name(repo_base="") + for tag, qual in self.additional_tags: + if qual == QUAL_NONE: + continue + yield self.image_name(tag=tag, repo_base="") + + @classmethod + def parse(cls, image_name): + if "/" in image_name: + base, rest = image_name.rsplit("/", 1) + else: + base = "" + rest = image_name + iname, tag = rest.split(":", 1) + tparts = tag.split("-", 3) + if len(tparts) < 3: + raise ValueError(f"too few tag components: {tag!r}") + return cls( + iname, + check_pkg_source(tparts[0]), + check_distro(tparts[1]), + check_arch(tparts[2]), + extra_tag=(tparts[3] if len(tparts) > 3 else ""), + repo_base=base, + ) + + +def generate_images(cli): + """Given full image names or a matrix of kind/pkg_source/distro_base/arch + values generate a list of target images to build/process. + """ + images = {} + for img in cli.image or []: + images[str(img)] = img + rc = RepoConfig(cli.repo_base, cli.repo_base_for) + for kind in cli.kind or []: + for pkg_source in cli.package_source or DEFAULT_PKG_SOURCES: + for distro_base in cli.distro_base or DEFAULT_DISTRO_BASES: + for arch in cli.arch or default_arches(): + timg = TargetImage( + kind, + pkg_source, + distro_base, + arch, + extra_tag=(cli.extra_tag or ""), + repo_base=rc.find_base(distro_base), + ) + images[str(timg)] = timg + return list(images.values()) + + +def add_special_tags(img): + """Certain images have special tags. Given an image, add general (non-FQIN) + tags to that image. + """ + # Most of the policy (as opposed to mechanism) resides here where we decide + # that certain images deserve some extra special tags. Mostly this serves + # to keep us compatible with older tagging schemes from earlier versions of + # the project. + if img.distro in [FEDORA, OPENSUSE]: + if img.arch == host_arch() and img.pkg_source == DEFAULT: + img.additional_tags.append((LATEST, QUAL_NONE)) + if img.arch == host_arch() and img.pkg_source == NIGHTLY: + img.additional_tags.append((NIGHTLY, QUAL_NONE)) + if img.arch == host_arch() and img.pkg_source == "default": + img.additional_tags.append((f"{img.distro}-{LATEST}", QUAL_DISTRO)) + if img.arch == host_arch() and img.pkg_source == "nightly": + img.additional_tags.append((f"{img.distro}-{NIGHTLY}", QUAL_DISTRO)) + + +def build(cli, target): + """Command to build images.""" + build_file = pathlib.Path(f"{cli.buildfile_prefix}{target.flat_name()}") + common_src = "./images/common" + common_dst = str(kind_source_dir(target.name) / ".common") + logger.debug("Copying common tree: %r -> %r", common_src, common_dst) + shutil.copytree(common_src, common_dst, dirs_exist_ok=True) + container_build(cli, target) + cid = container_id(cli, target) + with open(build_file, "w") as fh: + fh.write(f"{cid} {target.image_name()}\n") + + +def push(cli, target): + """Command to push images.""" + if cli.push_state == "rebuild": + build(cli, target) + if cli.push_state == "exists": + try: + container_id(cli, target) + except subprocess.CalledProcessError: + build(cli, target) + + push_name = target.image_name() + for tag in target.additional_tags: + if tag in ("latest", "nightly"): + push_name = target.image_name(tag=tag) + break + if tag.endswith(("-latest", "-nightly")): + push_name = target.image_name(tag=tag) + break + container_push(cli, push_name) + + +def print_buildfile(cli, target): + """Command to print build file names.""" + build_file = pathlib.Path(f"{cli.buildfile_prefix}{target.flat_name()}") + print(build_file) + + +def print_image(_, target): + """Command to print (fqin) image names.""" + print(str(target)) + + +def print_tags(cli, target): + """Command to print fqin image and additinal tag names.""" + for idx, name in enumerate( + target.all_names(baseless=cli.without_repo_bases) + ): + prefix = "" if idx == 0 else " " + print(f"{prefix}{name}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--verbose", + dest="log_level", + action="store_const", + const=logging.INFO, + default=logging.WARNING, + help="Emit verbose output", + ) + parser.add_argument( + "--debug", + dest="log_level", + action="store_const", + const=logging.DEBUG, + default=logging.WARNING, + help="Emit debug level output", + ) + parser.add_argument( + "--repo-base", + "-R", + help=( + "Common container registry repository base" + " (eg. quay.io/samba.org)" + ), + ) + parser.add_argument( + "--image", + "-i", + type=TargetImage.parse, + action="append", + help="Build precisely the named image (requires a FQIN)", + ) + parser.add_argument( + "--kind", + "-k", + type=check_kind, + action="append", + help="The kind of container to build (server, ad-server, etc...)", + ) + parser.add_argument( + "--distro-base", + "-d", + type=check_distro, + action="append", + help="The name of the base OS distribution to use.", + ) + parser.add_argument( + "--repo-base-for", + "-F", + type=check_repo_base_for, + action="append", + help=( + "Assign a custom repo base given a distro base" + "(like: --repo-base-for=centos=wonky.io/smb)" + ), + ) + parser.add_argument( + "--arch", + "-a", + type=check_arch, + action="append", + help="The name of the CPU architecture to build for", + ) + parser.add_argument( + "--package-source", + "-p", + type=check_pkg_source, + action="append", + help="Source of Samba packages to use", + ) + parser.add_argument( + "--container-engine", + help=( + "Expliclty specify the path to the container engine" + " (docker, podman, ...) to use" + ), + ) + parser.add_argument( + "--extra-tag", + help="Specify an extra tag extension. Handy for developers.", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Do not run build commands" + ) + parser.add_argument( + "--push-state", + choices=("exists", "rebuild"), + default="exists", + help=( + "Only push if a state is met:" + "exists - image exists; rebuild - image must be rebuilt." + ), + ) + parser.add_argument( + "--buildfile-prefix", + default=".build.", + help="Specify prefix for build status files", + ) + parser.add_argument( + "--extra-build-arg", + "-x", + action="append", + help="Extra argument to pass to container build command", + ) + parser.add_argument( + "--without-repo-bases", + "-w", + action="store_true", + help=( + "If an image has a repo base, also generate image names" + " without the repo base" + ), + ) + behaviors = parser.add_mutually_exclusive_group() + behaviors.add_argument( + "--push", + action="store_const", + dest="main_action", + const=push, + help="Push images", + ) + behaviors.add_argument( + "--print", + action="store_const", + dest="main_action", + const=print_image, + help="Print the image names selected", + ) + behaviors.add_argument( + "--print-tags", + action="store_const", + dest="main_action", + const=print_tags, + help="Print the image and additional tags selected", + ) + behaviors.add_argument( + "--print-buildfile", + action="store_const", + dest="main_action", + const=print_buildfile, + help="Print the names of build status files", + ) + cli = parser.parse_args() + + if os.environ.get("BUILD_IMAGE_DEBUG") in ("1", "yes"): + cli.log_level = logging.DEBUG + logging.basicConfig(level=cli.log_level) + + _action = cli.main_action if cli.main_action else build + imgs = [] + try: + imgs = generate_images(cli) + for img in imgs: + add_special_tags(img) + logger.info("Image %s, extra tags: %s", img, img.additional_tags) + _action(cli, img) + except subprocess.CalledProcessError as err: + logger.error("Failed command: %s", _cmd_to_str(err.cmd)) + sys.exit(err.returncode) + if not imgs: + logger.error("No images or image kinds supplied") + sys.exit(2) + + +if __name__ == "__main__": + main() From cbfca4f47bac73a72bfef9d77c746d42fbf13c83 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 31 May 2023 17:03:26 -0400 Subject: [PATCH 2/4] Makefile: replace swathes of makefile with use of hack/build-image Signed-off-by: John Mulligan --- Makefile | 221 +++++++++---------------------------------------------- 1 file changed, 33 insertions(+), 188 deletions(-) diff --git a/Makefile b/Makefile index 4e60abcf..202f6b1a 100644 --- a/Makefile +++ b/Makefile @@ -2,31 +2,14 @@ SELF = $(lastword $(MAKEFILE_LIST)) ROOT_DIR = $(realpath $(dir $(SELF))) CONTAINER_CMD ?= -ifeq ($(CONTAINER_CMD),) - CONTAINER_CMD:=$(shell podman version >/dev/null 2>&1 && echo podman) -endif -ifeq ($(CONTAINER_CMD),) - CONTAINER_CMD:=$(shell docker version >/dev/null 2>&1 && echo docker) -endif -# handle the case where podman is present but is (defaulting) to remote and is -# not not functioning correctly. Example: mac platform but no 'podman machine' -# vms are ready -ifeq ($(CONTAINER_CMD),) - CONTAINER_CMD:=$(shell podman --version >/dev/null 2>&1 && echo podman) -ifneq ($(CONTAINER_CMD),) -$(warning podman detected but 'podman version' failed. \ - this may mean your podman is set up for remote use, but is not working) -endif -endif - -BUILD_CMD:=$(CONTAINER_CMD) build $(BUILD_OPTS) -PUSH_CMD:=$(CONTAINER_CMD) push $(PUSH_OPTS) ALT_BIN=$(CURDIR)/.bin SHELLCHECK=$(shell command -v shellcheck || echo $(ALT_BIN)/shellcheck) GITLINT=$(shell command -v gitlint || echo $(ALT_BIN)/gitlint) YAMLLINT_CMD=$(shell command -v yamllint || echo $(ALT_BIN)/yamllint) -COMMON_DIR:=images/common +BUILD_IMAGE=$(ROOT_DIR)/hack/build-image --debug --without-repo-bases + + SERVER_DIR:=images/server AD_SERVER_DIR:=images/ad-server CLIENT_DIR:=images/client @@ -47,72 +30,17 @@ AD_SERVER_SOURCES=\ CLIENT_SRC_FILE=$(CLIENT_DIR)/Containerfile.$(SRC_OS_NAME) TOOLBOX_SRC_FILE=$(TOOLBOX_DIR)/Containerfile.$(SRC_OS_NAME) - BUILDFILE_PREFIX=.build -BUILDFILE_SERVER:=$(BUILDFILE_PREFIX).server -BUILDFILE_NIGHTLY_SERVER:=$(BUILDFILE_PREFIX).nightly-server -BUILDFILE_AD_SERVER:=$(BUILDFILE_PREFIX).ad-server -BUILDFILE_NIGHTLY_AD_SERVER:=$(BUILDFILE_PREFIX).nightly-ad-server -BUILDFILE_CLIENT:=$(BUILDFILE_PREFIX).client -BUILDFILE_TOOLBOX:=$(BUILDFILE_PREFIX).toolbox -OS_PREFIX=$(addsuffix -,$(OS_NAME)) -TAG=$(OS_PREFIX)latest -NIGHTLY_TAG=$(OS_PREFIX)nightly - - -SERVER_NAME=samba-server:$(TAG) -NIGHTLY_SERVER_NAME=samba-server:$(NIGHTLY_TAG) -AD_SERVER_NAME= samba-ad-server:$(TAG) -NIGHTLY_AD_SERVER_NAME=samba-ad-server:$(NIGHTLY_TAG) -CLIENT_NAME=samba-client:$(TAG) -NIGHTLY_CLIENT_NAME=samba-client:$(NIGHTLY_TAG) -TOOLBOX_NAME=samba-toolbox:$(TAG) -NIGHTLY_TOOLBOX_NAME=samba-toolbox:$(NIGHTLY_TAG) +BUILDFILE_SERVER=$(shell $(call _BUILD_KP,server,default,--print-buildfile)) +BUILDFILE_NIGHTLY_SERVER=$(shell $(call _BUILD_KP,server,nightly,--print-buildfile)) +BUILDFILE_AD_SERVER=$(shell $(call _BUILD_KP,ad-server,default,--print-buildfile)) +BUILDFILE_NIGHTLY_AD_SERVER=$(shell $(call _BUILD_KP,ad-server,nightly,--print-buildfile)) +BUILDFILE_CLIENT=$(shell $(call _BUILD_KP,client,default,--print-buildfile)) +BUILDFILE_TOOLBOX=$(shell $(call _BUILD_KP,toolbox,default,--print-buildfile)) REPO_BASE=quay.io/samba.org/ -SERVER_REPO_NAME=$(REPO_BASE)$(SERVER_NAME) -NIGHTLY_SERVER_REPO_NAME=$(REPO_BASE)$(NIGHTLY_SERVER_NAME) -AD_SERVER_REPO_NAME=$(REPO_BASE)$(AD_SERVER_NAME) -NIGHTLY_AD_SERVER_REPO_NAME=$(REPO_BASE)$(NIGHTLY_AD_SERVER_NAME) -CLIENT_REPO_NAME=$(REPO_BASE)$(CLIENT_NAME) -NIGHTLY_CLIENT_REPO_NAME=$(REPO_BASE)$(NIGHTLY_CLIENT_NAME) -TOOLBOX_REPO_NAME=$(REPO_BASE)$(TOOLBOX_NAME) -NIGHTLY_TOOLBOX_REPO_NAME=$(REPO_BASE)$(NIGHTLY_TOOLBOX_NAME) -BUILDFILE_PREFIX=.build -BUILDFILE_SERVER=$(BUILDFILE_PREFIX).$(OS_PREFIX)server -BUILDFILE_NIGHTLY_SERVER=$(BUILDFILE_PREFIX).$(OS_PREFIX)nightly-server -BUILDFILE_AD_SERVER=$(BUILDFILE_PREFIX).$(OS_PREFIX)ad-server -BUILDFILE_NIGHTLY_AD_SERVER=$(BUILDFILE_PREFIX).$(OS_PREFIX)nightly-ad-server -BUILDFILE_CLIENT=$(BUILDFILE_PREFIX).$(OS_PREFIX)client -BUILDFILE_NIGHTLY_CLIENT=$(BUILDFILE_PREFIX).$(OS_PREFIX)nightly-client -BUILDFILE_TOOLBOX=$(BUILDFILE_PREFIX).$(OS_PREFIX)toolbox -BUILDFILE_NIGHTLY_TOOLBOX=$(BUILDFILE_PREFIX).$(OS_PREFIX)nightly-toolbox - -HOST_ARCH:=$(shell arch) -HOST_ARCH:=$(subst x86_64,amd64,$(HOST_ARCH)) -HOST_ARCH:=$(subst aarch64,arm64,$(HOST_ARCH)) - -# build_fqin is a function macro for building a "Fully Qualified Image Name". -# Usage: $(call build_fqin,,,,,[]) -# base-name: the last part of the repo name eg. 'samba-server' -# pkg-source: source for samba packages (default or nightly) -# os-name: base os name -# arch: architecture of image (amd64, arm64, etc.) -# extra: (optional) an additional unique suffix for the tag -# typically meant for use by devs building custom images -build_fqin=$(REPO_BASE)$(1):$(2)-$(3)-$(4)$(if $(5),-$(5)) - -# get_imagename is a function macro for getting only the base image name -# without the tag part. -# Usage: $(call get_imagename,) -get_imagename=$(firstword $(subst :, ,$1)) - -# get_pkgsource is a function macro that, given an images name returns -# the name of the package source. Currently only understands the -# difference between default (os packages) and nightly (SIT packages). -# Usage: $(call, get_pkgsource,) -get_pkgsource=$(if $(findstring nightly,$1),nightly,default) +_BUILD_KP=$(BUILD_IMAGE) $(if $(CONTAINER_CMD),--container-engine=$(CONTAINER_CMD)) $(BI_PREFIX_ARGS) --kind=$1 --package-source=$2 --distro-base=$(SRC_OS_NAME) --repo-base=$(REPO_BASE) $(if $(BUILD_ARCH),--arch=$(BUILD_ARCH)) $3 arch_flag=$(strip $(if $(filter docker,$(CONTAINER_CMD)),\ @@ -128,26 +56,11 @@ build: build-server build-nightly-server build-ad-server build-client \ .PHONY: debug-vars debug-vars: @echo OS_NAME: $(OS_NAME) - @echo OS_PREFIX: $(OS_PREFIX) @echo TAG: $(TAG) @echo NIGHTLY_TAG: $(NIGHTLY_TAG) @echo SERVER_NAME: $(SERVER_NAME) - @echo SERVER_REPO_NAME: $(SERVER_REPO_NAME) - @echo NIGHTLY_SERVER_REPO_NAME: $(NIGHTLY_SERVER_REPO_NAME) @echo NIGHTLY_SERVER_NAME: $(NIGHTLY_SERVER_NAME) @echo AD_SERVER_NAME: $(AD_SERVER_NAME) - @echo AD_SERVER_REPO_NAME: $(AD_SERVER_REPO_NAME) - @echo NIGHTLY_AD_SERVER_NAME: $(NIGHTLY_AD_SERVER_NAME) - @echo NIGHTLY_AD_SERVER_NAME: $(NIGHTLY_AD_SERVER_NAME) - @echo NIGHTLY_AD_SERVER_REPO_NAME: $(NIGHTLY_AD_SERVER_REPO_NAME) - @echo CLIENT_NAME: $(CLIENT_NAME) - @echo CLIENT_REPO_NAME: $(CLIENT_REPO_NAME) - @echo NIGHTLY_CLIENT_NAME: $(NIGHTLY_CLIENT_NAME) - @echo NIGHTLY_CLIENT_REPO_NAME: $(NIGHTLY_CLIENT_REPO_NAME) - @echo TOOLBOX_NAME: $(TOOLBOX_NAME) - @echo TOOLBOX_REPO_NAME: $(TOOLBOX_REPO_NAME) - @echo NIGHTLY_TOOLBOX_NAME: $(NIGHTLY_TOOLBOX_NAME) - @echo NIGHTLY_TOOLBOX_REPO_NAME: $(NIGHTLY_TOOLBOX_REPO_NAME) @echo BUILDFILE_SERVER: $(BUILDFILE_SERVER) @echo BUILDFILE_AD_SERVER: $(BUILDFILE_AD_SERVER) @@ -155,7 +68,6 @@ debug-vars: @echo BUILDFILE_NIGHTLY_SERVER: $(BUILDFILE_NIGHTLY_SERVER) @echo BUILDFILE_CLIENT: $(BUILDFILE_CLIENT) @echo BUILDFILE_TOOLBOX: $(BUILDFILE_TOOLBOX) - @echo BUILDFILE_NIGHTLY_TOOLBOX: $(BUILDFILE_NIGHTLY_TOOLBOX) @echo SERVER_SRC_FILE: $(SERVER_SRC_FILE) @echo AD_SERVER_SRC_FILE: $(AD_SERVER_SRC_FILE) @@ -168,97 +80,55 @@ debug-vars: build-server: $(BUILDFILE_SERVER) .PHONY: build-server $(BUILDFILE_SERVER): Makefile $(SERVER_SRC_FILE) $(SERVER_SOURCES) - $(MAKE) _img_build \ - BUILD_ARGS="" \ - EXTRA_BUILD_ARGS="$(EXTRA_BUILD_ARGS)" \ - SHORT_NAME=$(SERVER_NAME) \ - REPO_NAME=$(SERVER_REPO_NAME) \ - SRC_FILE=$(SERVER_SRC_FILE) \ - DIR=$(SERVER_DIR) \ - BUILDFILE=$(BUILDFILE_SERVER) + $(call _BUILD_KP,server,default) $(EXTRA_BUILD_ARGS) push-server: build-server - $(PUSH_CMD) $(SERVER_REPO_NAME) + $(call _BUILD_KP,server,default,--push) .PHONY: push-server build-nightly-server: $(BUILDFILE_NIGHTLY_SERVER) .PHONY: build-nightly-server $(BUILDFILE_NIGHTLY_SERVER): Makefile $(SERVER_SRC_FILE) $(SERVER_SOURCES) - $(MAKE) _img_build \ - BUILD_ARGS="--build-arg=INSTALL_PACKAGES_FROM='samba-nightly'" \ - EXTRA_BUILD_ARGS="$(EXTRA_BUILD_ARGS)" \ - SHORT_NAME=$(NIGHTLY_SERVER_NAME) \ - REPO_NAME=$(NIGHTLY_SERVER_REPO_NAME) \ - SRC_FILE=$(SERVER_SRC_FILE) \ - DIR=$(SERVER_DIR) \ - BUILDFILE=$(BUILDFILE_NIGHTLY_SERVER) + $(call _BUILD_KP,server,nightly) $(EXTRA_BUILD_ARGS) push-nightly-server: build-nightly-server - $(PUSH_CMD) $(NIGHTLY_SERVER_REPO_NAME) + $(call _BUILD_KP,server,nightly,--push) .PHONY: push-nightly-server build-ad-server: $(BUILDFILE_AD_SERVER) .PHONY: build-ad-server $(BUILDFILE_AD_SERVER): Makefile $(AD_SERVER_SRC_FILE) $(AD_SERVER_SOURCES) - $(MAKE) _img_build \ - BUILD_ARGS="" \ - EXTRA_BUILD_ARGS="$(EXTRA_BUILD_ARGS)" \ - SHORT_NAME=$(AD_SERVER_NAME) \ - REPO_NAME=$(AD_SERVER_REPO_NAME) \ - SRC_FILE=$(AD_SERVER_SRC_FILE) \ - DIR=$(AD_SERVER_DIR) \ - BUILDFILE=$(BUILDFILE_AD_SERVER) + $(call _BUILD_KP,ad-server,default) $(EXTRA_BUILD_ARGS) push-ad-server: build-ad-server - $(PUSH_CMD) $(AD_SERVER_REPO_NAME) + $(call _BUILD_KP,ad-server,default,--push) .PHONY: push-ad-server build-nightly-ad-server: $(BUILDFILE_NIGHTLY_AD_SERVER) .PHONY: build-nightly-ad-server $(BUILDFILE_NIGHTLY_AD_SERVER): Makefile $(AD_SERVER_SRC_FILE) $(AD_SERVER_SOURCES) - $(MAKE) _img_build \ - BUILD_ARGS="--build-arg=INSTALL_PACKAGES_FROM='samba-nightly'" \ - EXTRA_BUILD_ARGS="$(EXTRA_BUILD_ARGS)" \ - SHORT_NAME=$(NIGHTLY_AD_SERVER_NAME) \ - REPO_NAME=$(NIGHTLY_AD_SERVER_REPO_NAME) \ - SRC_FILE=$(AD_SERVER_SRC_FILE) \ - DIR=$(AD_SERVER_DIR) \ - BUILDFILE=$(BUILDFILE_NIGHTLY_AD_SERVER) + $(call _BUILD_KP,ad-server,nightly) $(EXTRA_BUILD_ARGS) push-nightly-ad-server: build-nightly-ad-server - $(PUSH_CMD) $(NIGHTLY_AD_SERVER_REPO_NAME) + $(call _BUILD_KP,ad-server,nightly,--push) .PHONY: push-nightly-ad-server build-client: $(BUILDFILE_CLIENT) .PHONY: build-client $(BUILDFILE_CLIENT): Makefile $(CLIENT_SRC_FILE) - $(MAKE) _img_build \ - BUILD_ARGS="" \ - EXTRA_BUILD_ARGS="$(EXTRA_BUILD_ARGS)" \ - SHORT_NAME=$(CLIENT_NAME) \ - REPO_NAME=$(CLIENT_REPO_NAME) \ - SRC_FILE=$(CLIENT_SRC_FILE) \ - DIR=$(CLIENT_DIR) \ - BUILDFILE=$(BUILDFILE_CLIENT) + $(call _BUILD_KP,client,default) $(EXTRA_BUILD_ARGS) push-client: build-client - $(PUSH_CMD) $(CLIENT_REPO_NAME) + $(call _BUILD_KP,client,default,--push) .PHONY: push-client build-toolbox: $(BUILDFILE_TOOLBOX) .PHONY: build-toolbox $(BUILDFILE_TOOLBOX): Makefile $(TOOLBOX_SRC_FILE) - $(MAKE) _img_build \ - BUILD_ARGS="" \ - EXTRA_BUILD_ARGS="$(EXTRA_BUILD_ARGS)" \ - SHORT_NAME=$(TOOLBOX_NAME) \ - REPO_NAME=$(TOOLBOX_REPO_NAME) \ - SRC_FILE=$(TOOLBOX_SRC_FILE) \ - DIR=$(TOOLBOX_DIR) \ - BUILDFILE=$(BUILDFILE_TOOLBOX) + $(call _BUILD_KP,toolbox,default) $(EXTRA_BUILD_ARGS) push-toolbox: build-toolbox - $(PUSH_CMD) $(TOOLBOX_REPO_NAME) + $(call _BUILD_KP,toolbox,default,--push) .PHONY: push-toolbox @@ -268,11 +138,15 @@ test: test-server test-nightly-server .PHONY: test test-server: build-server - CONTAINER_CMD=$(CONTAINER_CMD) LOCAL_TAG=$(SERVER_NAME) tests/test-samba-container.sh + CONTAINER_CMD=$(CONTAINER_CMD) \ + LOCAL_TAG=$(shell cat $(BUILDFILE_SERVER) |cut -d' ' -f2) \ + tests/test-samba-container.sh .PHONY: test-server -test-nightly-server: build-nightly-server - CONTAINER_CMD=$(CONTAINER_CMD) LOCAL_TAG=$(NIGHTLY_SERVER_NAME) tests/test-samba-container.sh +test-nightly-server: $(BUILDFILE_NIGHTLY_SERVER) + CONTAINER_CMD=$(CONTAINER_CMD) \ + LOCAL_TAG=$(shell cat $(BUILDFILE_NIGHTLY_SERVER) |cut -d' ' -f2) \ + tests/test-samba-container.sh .PHONY: test-nightly-server @@ -295,48 +169,19 @@ check-gitlint: $(filter $(ALT_BIN)%,$(GITLINT)) $(GITLINT) -C .gitlint --commits origin/master.. lint .PHONY: check-gitlint -# _img_build is an "internal" rule to make the building of samba-container -# images regular and more "self documenting". A makefile.foo that includes -# this Makefile can add build rules using _img_build as a building block. -# -# The following arguments are expected to be supplied when "calling" this rule: -# BUILD_ARGS: the default build arguments -# EXTRA_BUILD_ARGS: build args supplied by the user at "runtime" -# SHORT_NAME: a local name for the image -# REPO_NAME: a global name for the image -# SRC_FILE: path to the Containerfile (Dockerfile) -# DIR: path to the directory holding image contents -# BUILDFILE: path to a temporary file tracking build state -_img_build: $(DIR)/.common - $(BUILD_CMD) \ - $(BUILD_ARGS) \ - $(call arch_flag) \ - $(EXTRA_BUILD_ARGS) \ - --tag $(SHORT_NAME) \ - --tag $(REPO_NAME) \ - --tag $(call build_fqin,$(call get_imagename,$(SHORT_NAME)),$(call get_pkgsource,$(SHORT_NAME)),$(SRC_OS_NAME),$(if $(BUILD_ARCH),$(BUILD_ARCH),$(HOST_ARCH)),$(EXTRA_TAG)) \ - -f $(SRC_FILE) \ - $(DIR) - $(CONTAINER_CMD) inspect -f '{{.Id}}' $(SHORT_NAME) > $(BUILDFILE) -.PHONY: _img_build - -$(DIR)/.common: $(COMMON_DIR) - $(RM) -r $(DIR)/.common - cp -r $(COMMON_DIR) $(DIR)/.common + +### Misc. Rules ### $(ALT_BIN)/%: $(CURDIR)/hack/install-tools.sh --$* $(ALT_BIN) - - -### Misc. Rules ### - clean: clean-buildfiles clean-altbin .PHONY: clean + clean-buildfiles: $(RM) $(BUILDFILE_PREFIX)* .PHONY: clean-buildfiles + clean-altbin: $(RM) -r $(ALT_BIN) .PHONY: clean-altbin - From 486f53389b23d68dfab7067c212e411d050a8126 Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Wed, 31 May 2023 17:03:53 -0400 Subject: [PATCH 3/4] Makefile.opensuse: remove unnecessary bits Signed-off-by: John Mulligan --- Makefile.opensuse | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Makefile.opensuse b/Makefile.opensuse index ff3bd1b3..ef71c9cf 100644 --- a/Makefile.opensuse +++ b/Makefile.opensuse @@ -1,14 +1,5 @@ include Makefile OS_NAME=opensuse -TAG=latest SERVER_SOURCES:=$(SERVER_DIR)/smb.conf -SERVER_NAME=samba-server:$(TAG) -AD_SERVER_NAME=samba-ad-server:$(TAG) -CLIENT_NAME=samba-client:$(TAG) -TOOLBOX_NAME=samba-toolbox:$(TAG) REPO_BASE=registry.opensuse.org/opensuse/ -SERVER_REPO_NAME=registry.opensuse.org/opensuse/samba-server:$(TAG) -AD_SERVER_REPO_NAME=registry.opensuse.org/opensuse/samba-ad-server:$(TAG) -CLIENT_REPO_NAME=registry.opensuse.org/opensuse/samba-client:$(TAG) -TOOLBOX_REPO_NAME=registry.opensuse.org/opensuse/samba-toolbox:$(TAG) From 087627f22da70b515e26d9a5c19d7d2f2293ba7f Mon Sep 17 00:00:00 2001 From: John Mulligan Date: Thu, 13 Jul 2023 19:09:58 -0400 Subject: [PATCH 4/4] toolbox: note an issue I found trying to build the centos toolbox The "fake" name used in the image only works if you build the container locally first or are running in our CI. Since this is not a good general solution, I put a comment here as a reminder it should be fixed later. Signed-off-by: John Mulligan --- images/toolbox/Containerfile.centos | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/images/toolbox/Containerfile.centos b/images/toolbox/Containerfile.centos index 40653638..974292f3 100644 --- a/images/toolbox/Containerfile.centos +++ b/images/toolbox/Containerfile.centos @@ -1,3 +1,8 @@ +# FIXME - this is not a real tag publicly available in the +# quay.io/samba.org/samba-client repository. This only works if you build +# the centos client locally first or acquire the image from a side channel. +# This needs to be converted to something public and/or configurable +# later. FROM quay.io/samba.org/samba-client:centos-latest MAINTAINER Shachar Sharon