diff --git a/Dockerfile b/Dockerfile index 811699e..343a52f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ COPY --from=datefudge /usr/bin/datefudge /usr/bin/datefudge COPY --from=resizefat32 /usr/bin/resizefat32 /usr/bin/resizefat32 RUN curl "https://github.com/gardenlinux/aws-kms-pkcs11/releases/download/latest/aws_kms_pkcs11-$(dpkg --print-architecture).so" -sLo "/usr/lib/$(uname -m)-linux-gnu/pkcs11/aws_kms_pkcs11.so" COPY builder /builder +RUN python3 -m pip install --break-system-packages -r "/builder/requirements.txt" --root-user-action ignore RUN mkdir /builder/cert COPY setup_namespace /usr/sbin/setup_namespace RUN curl -sSLf https://github.com/gardenlinux/seccomp_fake_xattr/releases/download/latest/seccomp_fake_xattr-$(uname -m).tar.gz \ diff --git a/build b/build index aebd69b..7f95e92 100755 --- a/build +++ b/build @@ -97,12 +97,9 @@ commit="$(./get_commit)" timestamp="$(./get_timestamp)" default_version="$(./get_version)" - if [ "$resolve_cname" = 1 ]; then arch="$("$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" dpkg --print-architecture)" - cname="$("$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" /builder/parse_features --feature-dir /builder/features --default-arch "$arch" --default-version "$default_version" --cname "$1")" - short_commit="$(head -c 8 <<< "$commit")" - echo "$cname-$short_commit" >&3 + "$container_engine" run --rm "${container_run_opts[@]}" "${container_mount_opts[@]}" "$container_image" gl-cname --feature-dir /builder/features --arch "$arch" --version "${default_version}-${commit}" "$1" exit 0 fi diff --git a/builder/Makefile b/builder/Makefile index cada2a7..1c8fbd2 100644 --- a/builder/Makefile +++ b/builder/Makefile @@ -10,10 +10,11 @@ export BASH_ENV := make_bash_env MAKEFLAGS += --no-builtin-rules -lastword = $(word $(words $1),$1) -prelastword = $(word $(words $1),_ $1) -cname_version = $(call lastword,$(subst -, ,$1)) -cname_arch = $(call prelastword,$(subst -, ,$1)) +cname_parts = $(subst _, , $(subst -, , $1)) +cname_gl_commit = $(lastword $(call cname_parts,$1)) +gl_version = $(lastword $(filter-out $(call cname_gl_commit,$1), $(call cname_parts,$1))) +cname_gl_version = $(call gl_version,$1)-$(call cname_gl_commit,$1) +cname_arch = $(lastword $(filter-out $(call gl_version,$1) $(call cname_gl_commit,$1), $(call cname_parts,$1))) define require_var = ifndef $1 @@ -24,11 +25,13 @@ endef required_vars := REPO COMMIT TIMESTAMP DEFAULT_VERSION TEMPFS_SIZE $(foreach var,$(required_vars),$(eval $(call require_var,$(var)))) -PARSE_FEATURES_ARGS := +GL_ALLOW_FRANKENSTEIN := ifdef ALLOW_FRANKENSTEIN -PARSE_FEATURES_ARGS := --allow-frankenstein +GL_ALLOW_FRANKENSTEIN := true endif +export GL_ALLOW_FRANKENSTEIN + SHORT_COMMIT := $(shell head -c 8 <<< '$(COMMIT)') DEFAULT_ARCH := $(shell dpkg --print-architecture) @@ -43,22 +46,22 @@ clean: .build/%.sentinel: true -.build/bootstrap-%-$(SHORT_COMMIT).tar: $$(shell ./make_repo_sentinel $$(REPO) $$(call cname_version,$$*)) +.build/bootstrap-%.tar: $$(shell ./make_repo_sentinel $$(REPO) $$(call gl_version,$$*)) target '$@' - info 'bootstrapping $*-$(SHORT_COMMIT)' + info 'bootstrapping $*' arch='$(call cname_arch,$*)' - version='$(call cname_version,$*)' + version='$(call gl_version,$*)' ./bootstrap "$$arch" "$$version" '$(REPO)' keyring.gpg '$@' -.build/%-$(SHORT_COMMIT).tar: .build/bootstrap-$$(call cname_arch,$$*)-$$(call cname_version,$$*)-$(SHORT_COMMIT).tar $(shell ./make_directory_sentinel features) $(shell ./make_directory_sentinel cert) +.build/%.tar: .build/bootstrap-$$(call cname_arch,$$*)-$$(call cname_gl_version,$$*).tar $(shell ./make_directory_sentinel features) $(shell ./make_directory_sentinel cert) target '$@' '$<' info 'configuring rootfs $*-$(SHORT_COMMIT)' - features="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' features)" - features_platforms="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' platforms)" - features_elements="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' elements)" - features_flags="$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$*' flags)" + features="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*' features)" + features_platforms="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'platforms)" + features_elements="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'elements)" + features_flags="$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$*'flags)" BUILDER_CNAME='$*' - BUILDER_VERSION='$(call cname_version,$*)' + BUILDER_VERSION='$(call gl_version,$*)' BUILDER_ARCH='$(call cname_arch,$*)' BUILDER_TIMESTAMP='$(TIMESTAMP)' BUILDER_COMMIT='$(COMMIT)' @@ -70,17 +73,17 @@ clean: ./configure '$(word 1,$^)' '$@' define artifact_template = -.build/%-$(SHORT_COMMIT).$1: $$$$(shell COMMIT=$(SHORT_COMMIT) ./make_get_image_dependencies '$$$$@') $$(shell ./make_directory_sentinel features) $$(shell ./make_directory_sentinel cert) +.build/%.$1: $$$$(shell COMMIT=$(SHORT_COMMIT) ./make_get_image_dependencies '$$$$@') $$(shell ./make_directory_sentinel features) $$(shell ./make_directory_sentinel cert) script='$$(word 1,$$^)' input='$$(word 2,$$^)' target '$$@' "$$$$input" info 'building $1 image $$*' - features="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' features)" - features_platforms="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' platforms)" - features_elements="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' elements)" - features_flags="$$$$(./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --cname '$$*' flags)" + features="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' features)" + features_platforms="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' platforms)" + features_elements="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' elements)" + features_flags="$$$$(gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --cname '$$*' flags)" BUILDER_CNAME='$$*' - BUILDER_VERSION='$$(call cname_version,$$*)' + BUILDER_VERSION='$$(call gl_version,$$*)' BUILDER_ARCH='$$(call cname_arch,$$*)' BUILDER_TIMESTAMP='$$(TIMESTAMP)' BUILDER_COMMIT='$$(COMMIT)' @@ -94,7 +97,7 @@ endef $(foreach artifact_rule,$(shell ./make_get_artifact_rules),$(eval $(call artifact_template,$(artifact_rule)))) -.build/%-$(SHORT_COMMIT).artifacts: $$(shell COMMIT=$(SHORT_COMMIT) ./make_list_build_artifacts '$$*') +.build/%.artifacts: $$(shell COMMIT=$(SHORT_COMMIT) DEFAULT_VERSION=$(DEFAULT_VERSION) NATIVE_ARCH=$(NATIVE_ARCH) ./make_list_build_artifacts '$$*') target '$@' echo -n > '$@' for f in $^; do @@ -102,7 +105,7 @@ $(foreach artifact_rule,$(shell ./make_get_artifact_rules),$(eval $(call artifac echo "$$(basename "$$f").log" | tee -a '$@' done -%: .build/$$(shell ./parse_features $(PARSE_FEATURES_ARGS) --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --default-version '$$(DEFAULT_VERSION)' --cname '$$*')-$(SHORT_COMMIT).artifacts +%: .build/$$(shell gl-features-parse --feature-dir features --default-arch '$$(DEFAULT_ARCH)' --default-version '$$(DEFAULT_VERSION)-$$(SHORT_COMMIT)' --cname '$$*').artifacts ln -f -s -r '$<' '.build/$*' # prevents match anything rule from applying to Makefile and image/convert scripts diff --git a/builder/make_get_image_dependencies b/builder/make_get_image_dependencies index f8789a0..f90edaf 100755 --- a/builder/make_get_image_dependencies +++ b/builder/make_get_image_dependencies @@ -7,7 +7,7 @@ exec 1>&2 # get longest chain of extensions, but not extensions starting with a number to prevent parsing minor version as extension extension="$(grep -E -o '(\.[a-z][a-zA-Z0-9\-_]*)*$' <<< "$1")" -artifact_base="${1%"-$COMMIT$extension"}" +artifact_base="${1%"$extension"}" cname="$(basename "$artifact_base")" [ "$extension" != ".raw" ] || extension= @@ -18,15 +18,15 @@ input= if [ -f "image$extension" ]; then script="image$extension" - input="$artifact_base-$COMMIT.tar" + input="$artifact_base.tar" fi if [ -f "convert$extension" ]; then script="convert$extension" - input="$artifact_base-$COMMIT.raw" + input="$artifact_base.raw" fi -IFS=',' read -r -a features < <(./parse_features --allow-frankenstein --feature-dir features --cname "$cname" features) +IFS=',' read -r -a features < <(gl-features-parse --feature-dir features --cname "$cname" features) for feature in "${features[@]}"; do if [ -s "features/$feature/image$extension" ]; then @@ -36,7 +36,7 @@ for feature in "${features[@]}"; do fi is_feature_script=1 script="features/$feature/image$extension" - input="$artifact_base-$COMMIT.tar" + input="$artifact_base.tar" fi if [ -s "features/$feature/convert$extension" ]; then @@ -46,7 +46,7 @@ for feature in "${features[@]}"; do fi is_feature_script=1 script="features/$feature/convert$extension" - input="$artifact_base-$COMMIT.raw" + input="$artifact_base.raw" fi # temporarily enable file globbing (+f) @@ -68,7 +68,7 @@ for feature in "${features[@]}"; do fi is_feature_script=1 script="$i" - input="$artifact_base-$COMMIT.${i##*~}" + input="$artifact_base.${i##*~}" done done diff --git a/builder/make_list_build_artifacts b/builder/make_list_build_artifacts index c75e993..ad3a780 100755 --- a/builder/make_list_build_artifacts +++ b/builder/make_list_build_artifacts @@ -5,21 +5,20 @@ shopt -s nullglob cname="$1" -IFS=',' read -r -a features < <(./parse_features --allow-frankenstein --feature-dir features --cname "$cname" features) - -artifacts=(".build/$cname-$COMMIT.tar" ".build/$cname-$COMMIT.release" ".build/$cname-$COMMIT.manifest" ".build/$cname-$COMMIT.sourcemanifest" ".build/$cname-$COMMIT.requirements") +IFS=',' read -r -a features < <(gl-features-parse --feature-dir features --default-arch "${NATIVE_ARCH}" --default-version "${DEFAULT_VERSION}-${COMMIT}" --cname "$cname" features) +artifacts=(".build/$cname.tar" ".build/$cname.release" ".build/$cname.manifest" ".build/$cname.sourcemanifest" ".build/$cname.requirements") for feature in "${features[@]}"; do for i in "features/$feature/"{image,convert}.*; do # get target artifact file extension, usually this is the image/convert script extension # except if the script extension is of the form filename.extA~extB in which case the artifact extension is .extA only extension="$(grep -E -o '(\.[a-z][a-zA-Z0-9\-_~]*)*$' <<< "$i")" - artifacts+=(".build/$cname-$COMMIT${extension%~*}") + artifacts+=(".build/$cname${extension%~*}") done done -if [ "${#artifacts[@]}" = 5 ] && [ -n "$(./parse_features --allow-frankenstein --feature-dir "features" --cname "$cname" platforms)" ]; then - artifacts+=(".build/$cname-$COMMIT.raw") +if [ "${#artifacts[@]}" = 4 ] && [ -n "$(gl-features-parse --feature-dir "features" --default-arch "${NATIVE_ARCH}" --default-version "${DEFAULT_VERSION}-${COMMIT}" --cname "$cname" platforms)" ]; then + artifacts+=(".build/$cname.raw") fi echo "${artifacts[@]}" diff --git a/builder/parse_features b/builder/parse_features deleted file mode 100755 index 146f6cd..0000000 --- a/builder/parse_features +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import argparse -import re -from functools import reduce -from glob import glob -import yaml -import networkx - -def main(): - parser = argparse.ArgumentParser() - - parser.add_argument("--feature-dir", default="features") - parser.add_argument("--features", type=lambda arg: set([f for f in arg.split(",") if f])) - parser.add_argument("--ignore", type=lambda arg: set([f for f in arg.split(",") if f]), default=set()) - parser.add_argument("--cname") - parser.add_argument("--arch") - parser.add_argument("--version") - parser.add_argument("--default-arch") - parser.add_argument("--default-version") - parser.add_argument("--allow-frankenstein", action="store_true") - - args_type_allowed = [ - "cname", - "cname_base", - "features", - "platforms", - "flags", - "elements", - "arch", - "version", - "graph" - ] - - parser.add_argument("type", nargs="?", choices=args_type_allowed, default="cname") - args = parser.parse_args() - - assert bool(args.features) ^ bool(args.cname), "please provide either `--features` or `--cname` argument" - - arch = None - version = None - - if args.cname: - search = re.search("^([a-z][a-zA-Z0-9_-]*?)(-(amd64|arm64)(-([a-z0-9.]+))?)?$", args.cname) - assert search, f"not a valid cname {args.cname}" - matches = search.groups() - input_cname_base = matches[0] - arch = matches[2] - version = matches[4] - input_features = reverse_cname_base(input_cname_base) - else: - input_features = args.features - - if args.arch: - arch = args.arch - - if args.version: - version = args.version - - if not arch and (args.type == "cname" or args.type == "arch"): - assert args.default_arch, "architecture not specified and no default architecture set" - arch = args.default_arch - if not version and (args.type == "cname" or args.type == "version"): - assert args.default_version, "version not specified and no default version set" - version = args.default_version - - if args.type == "arch": - print(arch) - sys.exit(0) - elif args.type == "version": - print(version) - sys.exit(0) - - feature_graph = read_feature_files(args.feature_dir) - - ignore_set = args.ignore - ignore_filter_func = lambda node : node not in ignore_set - feature_graph = networkx.subgraph_view(feature_graph, filter_node=ignore_filter_func) - - for feature in input_features: - assert feature in feature_graph.nodes(), f"feature {feature} not found" - - graph = filter_graph(feature_graph, input_features) - - features = reverse_sort_nodes(graph) - features_by_type = dict() - for type in [ "platform", "element", "flag" ]: - features_by_type[type] = [feature for feature in features if get_node_type(graph.nodes[feature]) == type] - - if len(features_by_type["platform"]) != 1: - print(f"{'warning' if args.allow_frankenstein else 'error'}: number of platforms is {len(features_by_type["platform"])}, but should be 1", file=sys.stderr) - if not args.allow_frankenstein: - print("hint: use --allow-frankenstein if you truly want to build with an unsupported number of platforms", file=sys.stderr) - sys.exit(1) - - sorted_features = sort_nodes(graph) - minimal_feature_set = get_minimal_feature_set(graph) - sorted_minimal_features = sort_set(minimal_feature_set, sorted_features) - cname_base = get_cname_base(sorted_minimal_features) - cname = f"{cname_base}-{arch}-{version}" - - if args.type == "cname_base": - print(cname_base) - elif args.type == "cname": - print(cname) - elif args.type == "features": - print(",".join(features)) - elif args.type == "platforms": - print(",".join(features_by_type["platform"])) - elif args.type == "elements": - print(",".join(features_by_type["element"])) - elif args.type == "flags": - print(",".join(features_by_type["flag"])) - elif args.type == "graph": - print(graph_as_mermaid_markup(cname_base, graph)) - -def graph_as_mermaid_markup(cname_base, graph): - """ - Generates a mermaid.js representation of the graph. - This is helpful to identify dependencies between features. - - Syntax docs: - https://mermaid.js.org/syntax/flowchart.html?id=flowcharts-basic-syntax - """ - markup = f"---\ntitle: Dependency Graph for Feature {cname_base}\n---\ngraph TD;\n" - for u,v in networkx.edges(graph): - markup += f" {u}-->{v};\n" - return markup - -def get_local_bin(name): - directory = os.path.dirname(os.path.realpath(__file__)) - return f"{directory}/{name}" - -def popen_read(command): - f = os.popen(command) - output = f.readline().rstrip("\n") - f.close() - return output - -def reverse_cname_base(cname): - cname = cname.replace("_", "-_") - return set(cname.split("-")) - -def get_cname_base(sorted_features): - return reduce(lambda a, b : a + ("-" if not b.startswith("_") else "") + b, sorted_features) - -def get_minimal_feature_set(graph): - return set([node for (node, degree) in graph.in_degree() if degree == 0]) - -def filter_graph(feature_graph, feature_set, ignore_excludes=False): - filter_set = set(feature_graph.nodes()) - filter_func = lambda node : node in filter_set - graph = networkx.subgraph_view(feature_graph, filter_node=filter_func) - graph_by_edge = dict() - for attr in [ "include", "exclude" ]: - edge_filter_func = (lambda attr : lambda a, b : graph.get_edge_data(a, b)["attr"] == attr)(attr) - graph_by_edge[attr] = networkx.subgraph_view(graph, filter_edge=edge_filter_func) - while True: - include_set = feature_set.copy() - for feature in feature_set: - include_set.update(networkx.descendants(graph_by_edge["include"], feature)) - filter_set = include_set - if ignore_excludes: - break - exclude_list = [] - for node in networkx.lexicographical_topological_sort(graph): - for exclude in graph_by_edge["exclude"].successors(node): - exclude_list.append(exclude) - if not exclude_list: - break - exclude = exclude_list[0] - assert exclude not in feature_set, f"excluding explicitly included feature {exclude}, unsatisfiable condition" - filter_set.remove(exclude) - assert (not graph_by_edge["exclude"].edges()) or ignore_excludes - return graph - -def read_feature_files(feature_dir): - feature_yaml_files = glob(f"{feature_dir}/*/info.yaml") - features = [parse_feature_yaml(i) for i in feature_yaml_files] - feature_graph = networkx.DiGraph() - for feature in features: - feature_graph.add_node(feature["name"], content=feature["content"]) - for node in feature_graph.nodes(): - node_features = get_node_features(feature_graph.nodes[node]) - for attr in node_features: - if attr not in [ "include", "exclude" ]: - continue - for ref in node_features[attr]: - assert os.path.isfile(f"{feature_dir}/{ref}/info.yaml"), f"feature {node} references feature {ref}, but {feature_dir}/{ref}/info.yaml does not exist" - feature_graph.add_edge(node, ref, attr=attr) - assert networkx.is_directed_acyclic_graph(feature_graph) - return feature_graph - -def parse_feature_yaml(feature_yaml_file): - assert os.path.basename(feature_yaml_file) == "info.yaml" - name = os.path.basename(os.path.dirname(feature_yaml_file)) - content = yaml.load(open(feature_yaml_file), Loader=yaml.FullLoader) - return { "name": name, "content": content } - -def sort_set(input_set, order_list): - return [item for item in order_list if item in input_set] - -def sort_key(graph, node): - prefix_map = { "platform": "0", "element": "1", "flag": "2" } - node_type = get_node_type(graph.nodes.get(node, {})) - prefix = prefix_map[node_type] - return f"{prefix}-{node}" - -def sort_nodes(graph): - key_lambda = lambda node : sort_key(graph, node) - return list(networkx.lexicographical_topological_sort(graph, key=key_lambda)) - -def reverse_sort_nodes(graph): - reverse_graph = graph.reverse() - assert networkx.is_directed_acyclic_graph(reverse_graph) - return sort_nodes(reverse_graph) - -def get_node_type(node): - return node.get("content", {}).get("type") - -def get_node_features(node): - return node.get("content", {}).get("features", {}) - -if __name__ == "__main__": - main() diff --git a/builder/requirements.txt b/builder/requirements.txt new file mode 100644 index 0000000..43c0da8 --- /dev/null +++ b/builder/requirements.txt @@ -0,0 +1,3 @@ +# Basic Python requirements for Garden Linux + +gardenlinux @ git+https://github.com/gardenlinux/python-gardenlinux-lib.git@0.10.9 diff --git a/pkg.list b/pkg.list index 288c8a7..eafaaab 100644 --- a/pkg.list +++ b/pkg.list @@ -23,10 +23,8 @@ openssl ostree ostree-boot python3 -python3-mako -python3-networkx -python3-pefile -python3-yaml +python3-setuptools +python3-pip qemu-utils squashfs-tools systemd diff --git a/setup_namespace b/setup_namespace index 1085abf..06bb0aa 100755 --- a/setup_namespace +++ b/setup_namespace @@ -4,6 +4,7 @@ set -eufo pipefail if [ "${1-}" = --second-stage ]; then shift + mount -t tmpfs -o size=4G tmpfs /tmp cleanup_permissions () { if [ -d /builder/.build ]; then