HACKATHON May 2025: integrate neon-pagecache-kmod

.dockerignore

@@ -2,5 +2,6 @@
 # Ignore build and test binaries.
 bin/
 testbin/
-
 rendered_manifests
+
+neonvm-kernel/neon-pagecache-kmod/linux-6.12.26

.github/workflows/vm-kernel.yaml

@@ -206,6 +206,8 @@ jobs:
 
       - name: git checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
 
       - name: set custom docker config directory
         uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "neonvm-kernel/neon-pagecache-kmod"]
+	path = neonvm-kernel/neon-pagecache-kmod
+	url = https://github.com/neondatabase-labs/christian-kernel-page-cache

Makefile

@@ -290,6 +290,7 @@ endif
 .PHONY: kernel
 kernel: ## Build linux kernel.
 	rm -f neonvm-kernel/vmlinuz; \
+	rm -rf neonvm-kernel/modules; \
 	linux_config=$$(ls neonvm-kernel/linux-config-*) \
 	kernel_version=$${linux_config##*-} \
 	iidfile=$$(mktemp /tmp/iid-XXXXXX); \
@@ -304,6 +305,7 @@ kernel: ## Build linux kernel.
 		neonvm-kernel; \
 	id=$$(docker create $$(cat $$iidfile)); \
 	docker cp $$id:/vmlinuz neonvm-kernel/vmlinuz; \
+	docker cp $$id:/modules neonvm-kernel/modules; \
 	docker rm -f $$id
 
 .PHONY: check-local-context

neonvm-kernel/.dockerignore

@@ -0,0 +1 @@
+neon-pagecache-kmod/linux-6.12.26

neonvm-kernel/.gitignore

@@ -1 +1,2 @@
 vmlinuz
+modules/

neonvm-kernel/Dockerfile

@@ -25,7 +25,8 @@ RUN apt-get update && apt-get -y install \
     python3 \
     cpio \
     zstd \
-    libncurses-dev
+    libncurses-dev \
+    git
 
 # Only require check-arg at this point, so the 'apt-get install' above is definitely cached.
 # We need to copy something in, otherwise 'docker buildx build' will just completely drop the
@@ -47,35 +48,55 @@ RUN set -e \
     && echo "unpacking kernel archive" \
     && tar --strip-components=1 -C linux-${KERNEL_VERSION} -xf linux-${KERNEL_VERSION}.tar.xz
 
-
+#RUN set -e \
+#    && echo "checking out neon-pagecache-kmod" \
+#    && git clone  https://github.com/neondatabase-labs/christian-kernel-page-cache.git \
+#    && cd christian-kernel-page-cache \
+#    && git checkout f54ff0d1bdf95a70693d37601d3d8c8867723e39
+# https://docs.docker.com/reference/dockerfile/#adding-files-from-a-git-repository
+#ADD https://github.com/neondatabase-labs/christian-kernel-page-cache.git#14574c3e255ef583d60ac3aa03ce7d84a9eb6bc8 /neon-pagecache-kmod
 
 ### Cross-compilation related steps
 
 # Build the kernel on amd64
-FROM build-deps AS build_amd64
+FROM build-deps AS build_amd64_kernel
 ARG KERNEL_VERSION
 ADD linux-config-amd64-${KERNEL_VERSION} linux-${KERNEL_VERSION}/.config
-RUN cd linux-${KERNEL_VERSION} && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- -j `nproc`
+RUN cd linux-${KERNEL_VERSION} \
+    && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- -j `nproc`
+FROM build_amd64_kernel AS build_amd64_kmod
+ARG KERNEL_VERSION
+COPY  ./neon-pagecache-kmod neon-pagecache-kmod
+RUN cd linux-${KERNEL_VERSION} \
+    && make ARCH=x86_64 CROSS_COMPILE=x86_64-linux-gnu- M=../neon-pagecache-kmod
 
 # Copy the kernel image to a separate step
 # Use alpine so that `cp` is available when loading custom kernels for the runner pod.
 # See the neonvm controller's pod creation logic for more detail.
 FROM --platform=linux/amd64 alpine:3.19.7@sha256:e5d0aea7f7d2954678a9a6269ca2d06e06591881161961ea59e974dff3f12377 AS kernel_amd64
 ARG KERNEL_VERSION
-COPY --from=build_amd64 /build/linux-${KERNEL_VERSION}/arch/x86/boot/bzImage /vmlinuz
+COPY --from=build_amd64_kernel /build/linux-${KERNEL_VERSION}/arch/x86/boot/bzImage /vmlinuz
+COPY --from=build_amd64_kmod /build/neon-pagecache-kmod/neon_pagecache.ko /modules/neon_pagecache.ko
 
 # Build the kernel on arm64
-FROM build-deps AS build_arm64
+FROM build-deps AS build_arm64_kernel
 ARG KERNEL_VERSION
 ADD linux-config-aarch64-${KERNEL_VERSION} linux-${KERNEL_VERSION}/.config
-RUN cd linux-${KERNEL_VERSION} && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -j `nproc`
+RUN cd linux-${KERNEL_VERSION} \
+    && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -j `nproc`
+FROM build_arm64_kernel AS build_arm64_kmod
+ARG KERNEL_VERSION
+COPY  ./neon-pagecache-kmod neon-pagecache-kmod
+RUN cd linux-${KERNEL_VERSION} \
+    && make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- M=../neon-pagecache-kmod
 
 # Copy the kernel image to a separate step
 # Use alpine so that `cp` is available when loading custom kernels for the runner pod.
 # See the neonvm controller's pod creation logic for more detail.
 FROM --platform=linux/arm64 alpine:3.19.7@sha256:e5d0aea7f7d2954678a9a6269ca2d06e06591881161961ea59e974dff3f12377 AS kernel_arm64
 ARG KERNEL_VERSION
-COPY --from=build_arm64 /build/linux-${KERNEL_VERSION}/arch/arm64/boot/Image /vmlinuz
+COPY --from=build_arm64_kernel /build/linux-${KERNEL_VERSION}/arch/arm64/boot/Image /vmlinuz
+COPY --from=build_arm64_kmod /build/neon-pagecache-kmod/neon_pagecache.ko /modules/neon_pagecache.ko
 
 # Dummy default target without target architecture
 FROM alpine:3.19.7@sha256:e5d0aea7f7d2954678a9a6269ca2d06e06591881161961ea59e974dff3f12377

neonvm-runner/Dockerfile

@@ -31,6 +31,7 @@ RUN apk add --no-cache \
 
 COPY --from=builder /runner /usr/bin/runner
 COPY neonvm-kernel/vmlinuz /vm/kernel/vmlinuz
+COPY neonvm-kernel/modules /vm/kernel/modules
 COPY neonvm-runner/ssh_config /etc/ssh/ssh_config
 # QEMU_EFI used only by runner running on the arm architecture
 RUN wget https://releases.linaro.org/components/kernel/uefi-linaro/16.02/release/qemu64/QEMU_EFI.fd -O /vm/QEMU_EFI_ARM.fd

neonvm-runner/cmd/disks.go

@@ -28,22 +28,32 @@
 	sshAuthorizedKeysDiskPath   = "/vm/images/ssh-authorized-keys.iso"
 	sshAuthorizedKeysMountPoint = "/vm/ssh"
 
+	kernelModulesDiskPath   = "/vm/images/kernel-modules.iso"
+
 	swapName = "swapdisk"
 )
 
 // setupVMDisks creates the disks for the VM and returns the appropriate QEMU args
 func setupVMDisks(
 	logger *zap.Logger,
-	diskCacheSettings string,
+	cfg *Config,
 	enableSSH bool,
 	swapSize *resource.Quantity,
 	extraDisks []vmv1.Disk,
 ) ([]string, error) {
 	var qemuCmd []string
 
-	qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=rootdisk,file=%s,if=virtio,media=disk,index=0,%s", rootDiskPath, diskCacheSettings))
+	qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=rootdisk,file=%s,if=virtio,media=disk,index=0,%s", rootDiskPath, cfg.diskCacheSettings))
 	qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=runtime,file=%s,if=virtio,media=cdrom,readonly=on,cache=none", runtimeDiskPath))
 
+	{
+		name := "kernel-modules"
+		if err := createISO9660FromPath(logger, name, kernelModulesDiskPath, cfg.kernelModulesPath); err != nil {
+			return nil, fmt.Errorf("Failed to create ISO9660 image: %w", err)
+		}
+		qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=cdrom,cache=none", name, kernelModulesDiskPath))
+	}
+
 	if enableSSH {
 		name := "ssh-authorized-keys"
 		if err := createISO9660FromPath(logger, name, sshAuthorizedKeysDiskPath, sshAuthorizedKeysMountPoint); err != nil {
@@ -58,7 +68,7 @@
 		if err := createSwap(dPath, swapSize); err != nil {
 			return nil, fmt.Errorf("Failed to create swap disk: %w", err)
 		}
-		qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s,discard=unmap", swapName, dPath, diskCacheSettings))
+		qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s,discard=unmap", swapName, dPath, cfg.diskCacheSettings))
 	}
 
 	for _, disk := range extraDisks {
@@ -73,7 +83,7 @@
 			if disk.EmptyDisk.Discard {
 				discard = ",discard=unmap"
 			}
-			qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s%s", disk.Name, dPath, diskCacheSettings, discard))
+			qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s%s", disk.Name, dPath, cfg.diskCacheSettings, discard))
 		case disk.ConfigMap != nil || disk.Secret != nil:
 			dPath := fmt.Sprintf("%s/%s.iso", mountedDiskPath, disk.Name)
 			mnt := fmt.Sprintf("/vm/mounts%s", disk.MountPath)
@@ -173,6 +183,12 @@
 	mounts := []string{
 		"set -euo pipefail",
 	}
+
+	// {
+	// 	mounts = append(mounts, "/neonvm/bin/mkdir -p /mnt/kernel-modules")
+	// 	mounts = append(mounts, "/neonvm/bin/mount  -t iso9660 -o ro,mode=0644 $(/neonvm/bin/blkid -L kernel-modules) /mnt/kernel-modules")
+	// }
+
 	if enableSSH {
 		mounts = append(mounts, "/neonvm/bin/mkdir -p /mnt/ssh")
 		mounts = append(mounts, "/neonvm/bin/mount  -t iso9660 -o ro,mode=0644 $(/neonvm/bin/blkid -L ssh-authorized-keys) /mnt/ssh")

neonvm-runner/cmd/main.go

@@ -42,9 +42,10 @@ const (
 	qemuBinX8664 = "qemu-system-x86_64"
 	qemuImgBin   = "qemu-img"
 
-	architectureArm64 = "arm64"
-	architectureAmd64 = "amd64"
-	defaultKernelPath = "/vm/kernel/vmlinuz"
+	architectureArm64        = "arm64"
+	architectureAmd64        = "amd64"
+	defaultKernelPath        = "/vm/kernel/vmlinuz"
+	defaultKernelModulesPath = "/vm/kernel/modules"
 
 	qmpUnixSocketForSigtermHandler = "/vm/qmp-sigterm.sock"
 	logSerialSocket                = "/vm/log.sock"
@@ -104,6 +105,7 @@ type Config struct {
 	vmSpecDump           string
 	vmStatusDump         string
 	kernelPath           string
+	kernelModulesPath    string
 	appendKernelCmdline  string
 	skipCgroupManagement bool
 	diskCacheSettings    string
@@ -122,6 +124,7 @@ func newConfig(logger *zap.Logger) *Config {
 		vmSpecDump:           "",
 		vmStatusDump:         "",
 		kernelPath:           defaultKernelPath,
+		kernelModulesPath:    defaultKernelModulesPath,
 		appendKernelCmdline:  "",
 		skipCgroupManagement: false,
 		diskCacheSettings:    "cache=none",
@@ -136,6 +139,8 @@ func newConfig(logger *zap.Logger) *Config {
 		"Base64 gzip compressed VirtualMachine json status")
 	flag.StringVar(&cfg.kernelPath, "kernelpath", cfg.kernelPath,
 		"Override path for kernel to use")
+	flag.StringVar(&cfg.kernelModulesPath, "kernelmodulespath", cfg.kernelModulesPath,
+		"Override path for kernel modules to use")
 	flag.StringVar(&cfg.appendKernelCmdline, "appendKernelCmdline",
 		cfg.appendKernelCmdline, "Additional kernel command line arguments")
 	flag.BoolVar(&cfg.skipCgroupManagement, "skip-cgroup-management",
@@ -321,7 +326,7 @@ func buildQEMUCmd(
 		"-device", "virtserialport,chardev=log,name=tech.neon.log.0",
 	}
 
-	qemuDiskArgs, err := setupVMDisks(logger, cfg.diskCacheSettings, enableSSH, swapSize, vmSpec.Disks)
+	qemuDiskArgs, err := setupVMDisks(logger, cfg, enableSSH, swapSize, vmSpec.Disks)
 	if err != nil {
 		return nil, err
 	}

vm-builder/files/vminit

@@ -25,6 +25,12 @@ mount -t proc  proc  /proc
 mount -t sysfs sysfs /sys
 mount -t cgroup2 cgroup2 /sys/fs/cgroup
 
+# load kernel modules
+mkdir -p /lib/modules/$(uname -r)/kernel/extra/
+mount -t iso9660 -o ro,mode=0644 /dev/vdc /lib/modules/$(uname -r)/kernel/extra/
+## XXX avoid hard-coding list of kernel modules; supply list as a text file in the ISO, produced by neonvm-kernel/Dockerfile ?
+modprobe neon_pagecache
+
 # change kernel log level to DEBUG
 echo 7 > /proc/sys/kernel/printk