krata: rework cross-compilation

hack/build/cargo.sh

@@ -3,11 +3,22 @@ set -e
 
 TOOLS_DIR="$(dirname "${0}")"
 RUST_TARGET="$("${TOOLS_DIR}/target.sh")"
+CROSS_COMPILE="$("${TOOLS_DIR}/cross-compile.sh")"
 
-if [ "${RUST_LIBC}" = "musl" ] && [ -f "/etc/alpine-release" ]
+if [ "${TARGET_LIBC}" = "musl" ] && [ -f "/etc/alpine-release" ]
 then
   export RUSTFLAGS="-Ctarget-feature=-crt-static"
 fi
 
+if [ -z "${CARGO}" ]
+then
+  if [ "${CROSS_COMPILE}" = "1" ] && command -v cross > /dev/null
+  then
+    CARGO="cross"
+  else
+    CARGO="cargo"
+  fi
+fi
+
 export CARGO_BUILD_TARGET="${RUST_TARGET}"
-exec cargo "${@}"
+exec "${CARGO}" "${@}"

hack/build/cross-compile.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+TOOLS_DIR="$(dirname "${0}")"
+
+RUST_TARGET="$("${TOOLS_DIR}/target.sh")"
+TARGET_ARCH="$(echo "${RUST_TARGET}" | awk -F '-' '{print $1}')"
+HOST_ARCH="$(uname -m)"
+
+if [ "${HOST_ARCH}" != "${TARGET_ARCH}" ]
+then
+  echo "1"
+else
+  echo "0"
+fi

hack/build/target.sh

@@ -1,29 +1,57 @@
 #!/bin/sh
 set -e
 
-if [ -z "${RUST_LIBC}" ]
+if [ -z "${TARGET_LIBC}" ] || [ "${KRATA_TARGET_IGNORE_LIBC}" = "1" ]
 then
-  RUST_LIBC="gnu"
+  TARGET_LIBC="gnu"
+fi
+
+if [ -z "${TARGET_ARCH}" ]
+then
+  TARGET_ARCH="$(uname -m)"
 fi
 
 if [ -z "${RUST_TARGET}" ]
 then
-  HOST_ARCH="$(uname -m)"
-  if [ "${HOST_ARCH}" = "x86_64" ]
+  if [ "${TARGET_ARCH}" = "x86_64" ]
   then
-    RUST_TARGET="x86_64-unknown-linux-${RUST_LIBC}"
+    RUST_TARGET="x86_64-unknown-linux-${TARGET_LIBC}"
   fi
 
-  if [ "${HOST_ARCH}" = "aarch64" ]
+  if [ "${TARGET_ARCH}" = "aarch64" ]
   then
-    RUST_TARGET="aarch64-unknown-linux-${RUST_LIBC}"
+    RUST_TARGET="aarch64-unknown-linux-${TARGET_LIBC}"
   fi
 fi
 
-if [ -z "${RUST_TARGET}" ]
+if [ -z "${C_TARGET}" ]
 then
-  echo "ERROR: Unable to determine RUST_TARGET, your architecture may not be supported by krata." > /dev/stderr
-  exit 1
+  if [ "${TARGET_ARCH}" = "x86_64" ]
+  then
+    C_TARGET="x86_64-linux-${TARGET_LIBC}"
+  fi
+
+  if [ "${TARGET_ARCH}" = "aarch64" ]
+  then
+    C_TARGET="aarch64-linux-${TARGET_LIBC}"
+  fi
 fi
 
-echo "${RUST_TARGET}"
+if [ "${KRATA_TARGET_C_MODE}" = "1" ]
+then
+  if [ -z "${C_TARGET}" ]
+  then
+    echo "ERROR: Unable to determine C_TARGET, your architecture may not be supported by krata." > /dev/stderr
+    exit 1
+  fi
+
+  echo "${C_TARGET}"
+else
+  if [ -z "${RUST_TARGET}" ]
+  then
+    echo "ERROR: Unable to determine RUST_TARGET, your architecture may not be supported by krata." > /dev/stderr
+    exit 1
+  fi
+
+  echo "${RUST_TARGET}"
+fi

hack/ci/install-deps.sh

@@ -6,3 +6,4 @@ sudo apt-get install -y \
     build-essential libssl-dev libelf-dev musl-dev \
     flex bison bc protobuf-compiler musl-tools qemu-utils
 sudo gem install --no-document fpm
+cargo install cross

hack/dist/apk.sh

@@ -4,7 +4,7 @@ set -e
 # shellcheck source-path=SCRIPTDIR source=common.sh
 . "$(dirname "${0}")/common.sh"
 
-export RUST_LIBC="musl"
+export TARGET_LIBC="musl"
 KRATA_SYSTAR_OPENRC=1 "${KRATA_DIR}/hack/dist/systar.sh"
 
 KRATA_VERSION="$("${KRATA_DIR}/hack/dist/version.sh")"
@@ -22,4 +22,4 @@ fpm -s tar -t apk \
   --description "Krata Hypervisor" \
   --url "https://krata.dev" \
   --maintainer "Edera Team <contact@edera.dev>" \
-  "${OUTPUT_DIR}/system-openrc.tgz"
+  "${OUTPUT_DIR}/system-openrc-${TARGET_ARCH}.tgz"

hack/dist/bundle.sh

@@ -9,15 +9,18 @@ then
   KRATA_KERNEL_BUILD_JOBS="2"
 fi
 
-BUNDLE_TAR="${OUTPUT_DIR}/bundle-systemd.tgz"
+TARGET_ARCH="$("${KRATA_DIR}/hack/build/arch.sh")"
+BUNDLE_TAR="${OUTPUT_DIR}/bundle-systemd-${TARGET_ARCH}.tgz"
 rm -f "${BUNDLE_TAR}"
 BUNDLE_DIR="$(mktemp -d /tmp/krata-bundle.XXXXXXXXXXXXX)"
 BUNDLE_DIR="${BUNDLE_DIR}/krata"
 mkdir -p "${BUNDLE_DIR}"
+
+./hack/build/cargo.sh build --release --bin kratad --bin kratanet --bin kratactl
+
+RUST_TARGET="$(./hack/build/target.sh)"
 for X in kratad kratanet kratactl
 do
-  ./hack/build/cargo.sh build --release --bin "${X}"
-  RUST_TARGET="$(./hack/build/target.sh)"
   cp "${KRATA_DIR}/target/${RUST_TARGET}/release/${X}" "${BUNDLE_DIR}/${X}"
 done
 ./hack/initrd/build.sh
@@ -28,8 +31,8 @@ fi
 
 cd "${BUNDLE_DIR}"
 
-cp "${KRATA_DIR}/target/initrd/initrd" initrd
-cp "${KRATA_DIR}/target/kernel/kernel" kernel
+cp "${KRATA_DIR}/target/initrd/initrd-${TARGET_ARCH}" initrd
+cp "${KRATA_DIR}/target/kernel/kernel-${TARGET_ARCH}" kernel
 cp "${KRATA_DIR}/resources/systemd/kratad.service" kratad.service
 cp "${KRATA_DIR}/resources/systemd/kratanet.service" kratanet.service
 cp "${KRATA_DIR}/resources/bundle/install.sh" install.sh

hack/dist/deb.sh

@@ -7,18 +7,19 @@ set -e
 "${KRATA_DIR}/hack/dist/systar.sh"
 
 KRATA_VERSION="$("${KRATA_DIR}/hack/dist/version.sh")"
-TARGET_ARCH="$(KRATA_ARCH_ALT_NAME=1 "${KRATA_DIR}/hack/build/arch.sh")"
+TARGET_ARCH_STANDARD="$(KRATA_ARCH_ALT_NAME=0 "${KRATA_DIR}/hack/build/arch.sh")"
+TARGET_ARCH_DEBIAN="$(KRATA_ARCH_ALT_NAME=1 "${KRATA_DIR}/hack/build/arch.sh")"
 
 cd "${OUTPUT_DIR}"
 
-rm -f "krata_${KRATA_VERSION}_${TARGET_ARCH}.deb"
+rm -f "krata_${KRATA_VERSION}_${TARGET_ARCH_DEBIAN}.deb"
 
 fpm -s tar -t deb \
   --name krata \
   --license agpl3 \
   --version "${KRATA_VERSION}" \
-  --architecture "${TARGET_ARCH}" \
-  --depends "xen-system-${TARGET_ARCH}" \
+  --architecture "${TARGET_ARCH_DEBIAN}" \
+  --depends "xen-system-${TARGET_ARCH_DEBIAN}" \
   --description "Krata Hypervisor" \
   --url "https://krata.dev" \
   --maintainer "Edera Team <contact@edera.dev>" \
@@ -27,4 +28,4 @@ fpm -s tar -t deb \
   --deb-systemd "${KRATA_DIR}/resources/systemd/kratanet.service" \
   --deb-systemd-enable \
   --deb-systemd-auto-start \
-  "${OUTPUT_DIR}/system-systemd.tgz"
+  "${OUTPUT_DIR}/system-systemd-${TARGET_ARCH_STANDARD}.tgz"

hack/dist/systar.sh

@@ -12,11 +12,12 @@ then
   SYSTAR_VARIANT="openrc"
 fi
 
-SYSTAR="${OUTPUT_DIR}/system-${SYSTAR_VARIANT}.tgz"
+TARGET_ARCH="$("${KRATA_DIR}/hack/build/arch.sh")"
+SYSTAR="${OUTPUT_DIR}/system-${SYSTAR_VARIANT}-${TARGET_ARCH}.tgz"
 rm -f "${SYSTAR}"
 SYSTAR_DIR="$(mktemp -d /tmp/krata-systar.XXXXXXXXXXXXX)"
 cd "${SYSTAR_DIR}"
-tar xf "${OUTPUT_DIR}/bundle-systemd.tgz"
+tar xf "${OUTPUT_DIR}/bundle-systemd-${TARGET_ARCH}.tgz"
 
 mkdir sys
 cd sys

hack/initrd/build.sh

@@ -6,7 +6,9 @@ cd "$(dirname "${REAL_SCRIPT}")/../.."
 KRATA_DIR="${PWD}"
 cd "${KRATA_DIR}"
 
-export RUST_LIBC="musl"
+TARGET_ARCH="$(./hack/build/arch.sh)"
+
+export TARGET_LIBC="musl"
 RUST_TARGET="$(./hack/build/target.sh)"
 export RUSTFLAGS="-Ctarget-feature=+crt-static"
 
@@ -16,5 +18,5 @@ cp "target/${RUST_TARGET}/release/krataguest" "${INITRD_DIR}/init"
 chmod +x "${INITRD_DIR}/init"
 cd "${INITRD_DIR}"
 mkdir -p "${KRATA_DIR}/target/initrd"
-find . | cpio -R 0:0 --ignore-devno --renumber-inodes -o -H newc --quiet > "${KRATA_DIR}/target/initrd/initrd"
+find . | cpio -R 0:0 --ignore-devno --renumber-inodes -o -H newc --quiet > "${KRATA_DIR}/target/initrd/initrd-${TARGET_ARCH}"
 rm -rf "${INITRD_DIR}"

hack/kernel/build.sh

@@ -8,9 +8,21 @@ KERNEL_DIR="${KRATA_DIR}/kernel"
 
 cd "${KRATA_DIR}"
 
+TARGET_ARCH_STANDARD="$(KRATA_ARCH_KERNEL_NAME=0 ./hack/build/arch.sh)"
+TARGET_ARCH_KERNEL="$(KRATA_ARCH_KERNEL_NAME=1 ./hack/build/arch.sh)"
+C_TARGET="$(KRATA_TARGET_C_MODE=1 KRATA_TARGET_IGNORE_LIBC=1 ./hack/build/target.sh)"
+IS_CROSS_COMPILE="$(./hack/build/cross-compile.sh)"
+
+if [ "${IS_CROSS_COMPILE}" = "1" ]
+then
+  CROSS_COMPILE_MAKE="CROSS_COMPILE=${C_TARGET}-"
+else
+  CROSS_COMPILE_MAKE="CROSS_COMPILE="
+fi
+
 # shellcheck source-path=SCRIPTDIR source=../../kernel/config.sh
 . "${KERNEL_DIR}/config.sh"
-KERNEL_SRC="${KERNEL_DIR}/linux-${KERNEL_VERSION}"
+KERNEL_SRC="${KERNEL_DIR}/linux-${KERNEL_VERSION}-${TARGET_ARCH_STANDARD}"
 
 if [ -z "${KRATA_KERNEL_BUILD_JOBS}" ]
 then
@@ -20,42 +32,45 @@ fi
 if [ ! -f "${KERNEL_SRC}/Makefile" ]
 then
   rm -rf "${KERNEL_SRC}"
+  mkdir -p "${KERNEL_SRC}"
   curl --progress-bar -L -o "${KERNEL_SRC}.txz" "${KERNEL_SRC_URL}"
-  tar xf "${KERNEL_SRC}.txz" -C "${KERNEL_DIR}"
+  tar xf "${KERNEL_SRC}.txz" --strip-components 1 -C "${KERNEL_SRC}"
   rm "${KERNEL_SRC}.txz"
 fi
 
 OUTPUT_DIR="${KRATA_DIR}/target/kernel"
 mkdir -p "${OUTPUT_DIR}"
 
-TARGET_ARCH="$(KRATA_ARCH_KERNEL_NAME=1 ./hack/build/arch.sh)"
-
-KERNEL_CONFIG_FILE="${KERNEL_DIR}/krata-${TARGET_ARCH}.config"
+KERNEL_CONFIG_FILE="${KERNEL_DIR}/krata-${TARGET_ARCH_STANDARD}.config"
 
 if [ ! -f "${KERNEL_CONFIG_FILE}" ]
 then
-  echo "ERROR: kernel config file not found for ${TARGET_ARCH}" > /dev/stderr
+  echo "ERROR: kernel config file not found for ${TARGET_ARCH_STANDARD}" > /dev/stderr
   exit 1
 fi
 
 cp "${KERNEL_CONFIG_FILE}" "${KERNEL_SRC}/.config"
-make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH}" olddefconfig
+make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH_KERNEL}" "${CROSS_COMPILE_MAKE}" olddefconfig
 
-if [ "${TARGET_ARCH}" = "x86_64" ]
+IMAGE_TARGET="bzImage"
+
+if [ "${TARGET_ARCH_STANDARD}" = "x86_64" ]
 then
-  make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH}" -j"${KRATA_KERNEL_BUILD_JOBS}" bzImage
-elif [ "${TARGET_ARCH}" = "arm64" ]
+  IMAGE_TARGET="bzImage"
+elif [ "${TARGET_ARCH_STANDARD}" = "aarch64" ]
 then
-  make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH}" -j"${KRATA_KERNEL_BUILD_JOBS}" Image.gz
+  IMAGE_TARGET="Image.gz"
 fi
 
-if [ "${TARGET_ARCH}" = "x86_64" ]
+make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH_KERNEL}" -j"${KRATA_KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" "${IMAGE_TARGET}"
+
+if [ "${TARGET_ARCH_STANDARD}" = "x86_64" ]
 then
-  cp "${KERNEL_SRC}/arch/x86/boot/bzImage" "${OUTPUT_DIR}/kernel"
-elif [ "${TARGET_ARCH}" = "arm64" ]
+  cp "${KERNEL_SRC}/arch/x86/boot/bzImage" "${OUTPUT_DIR}/kernel-${TARGET_ARCH_STANDARD}"
+elif [ "${TARGET_ARCH_STANDARD}" = "aarch64" ]
 then
-  cp "${KERNEL_SRC}/arch/arm64/boot/Image.gz" "${OUTPUT_DIR}/kernel"
+  cp "${KERNEL_SRC}/arch/arm64/boot/Image.gz" "${OUTPUT_DIR}/kernel-${TARGET_ARCH_STANDARD}"
 else
-  echo "ERROR: unable to determine what file is the vmlinuz for ${TARGET_ARCH}" > /dev/stderr
+  echo "ERROR: unable to determine what file is the vmlinuz for ${TARGET_ARCH_STANDARD}" > /dev/stderr
   exit 1
 fi

hack/kernel/clean.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -e
-
-REAL_SCRIPT="$(realpath "${0}")"
-cd "$(dirname "${REAL_SCRIPT}")/../.."
-KRATA_DIR="${PWD}"
-KERNEL_DIR="${KRATA_DIR}/kernel"
-
-# shellcheck disable=SC2010
-for ITEM in $(ls "${KERNEL_DIR}" | grep "^linux-")
-do
-  rm -rf "${KERNEL_DIR:?}/${ITEM}"
-done

hack/os/build.sh

@@ -7,20 +7,31 @@ cd "$(dirname "${REAL_SCRIPT}")/../.."
 ./hack/dist/apk.sh
 KRATA_VERSION="$(./hack/dist/version.sh)"
 TARGET_ARCH="$(./hack/build/arch.sh)"
+TARGET_ARCH_ALT="$(KRATA_ARCH_KERNEL_NAME=1 ./hack/build/arch.sh)"
+CROSS_COMPILE="$(./hack/build/cross-compile.sh)"
 
 TARGET_DIR="${PWD}/target"
 TARGET_OS_DIR="${TARGET_DIR}/os"
 mkdir -p "${TARGET_OS_DIR}"
-cp "${TARGET_DIR}/dist/krata_${KRATA_VERSION}_${TARGET_ARCH}.apk" "${TARGET_OS_DIR}/krata.apk"
+cp "${TARGET_DIR}/dist/krata_${KRATA_VERSION}_${TARGET_ARCH}.apk" "${TARGET_OS_DIR}/krata-${TARGET_ARCH}.apk"
 
-DOCKER_INTERACTIVE_FLAGS=""
+DOCKER_FLAGS=""
 if [ -t 0 ]
 then
- DOCKER_INTERACTIVE_FLAGS="-it"
+ DOCKER_FLAGS="-it"
 fi
 
-docker run --rm --privileged -v "${PWD}:/mnt" ${DOCKER_INTERACTIVE_FLAGS} alpine:latest "/mnt/os/internal/stage1.sh"
-sudo chown "${USER}:${GROUP}" "${TARGET_OS_DIR}/rootfs.tar"
+if [ "${CROSS_COMPILE}" = "1" ]
+then
+  DOCKER_FLAGS="${DOCKER_FLAGS} --platform linux/${TARGET_ARCH_ALT}"
+  docker run --privileged --rm tonistiigi/binfmt --install all
+fi
+
+ROOTFS="${TARGET_OS_DIR}/rootfs-${TARGET_ARCH}.tar"
+
+# shellcheck disable=SC2086
+docker run --rm --privileged -v "${PWD}:/mnt" ${DOCKER_FLAGS} alpine:latest "/mnt/os/internal/stage1.sh" "${TARGET_ARCH}"
+sudo chown "${USER}:${GROUP}" "${ROOTFS}"
 sudo modprobe nbd
 
 next_nbd_device() {
@@ -42,10 +53,10 @@ then
   exit 1
 fi
 
-OS_IMAGE="${TARGET_OS_DIR}/krata.qcow2"
+OS_IMAGE="${TARGET_OS_DIR}/krata-${TARGET_ARCH}.qcow2"
 EFI_PART="${NBD_DEVICE}p1"
 ROOT_PART="${NBD_DEVICE}p2"
-ROOT_DIR="${TARGET_OS_DIR}/root"
+ROOT_DIR="${TARGET_OS_DIR}/root-${TARGET_ARCH}"
 EFI_DIR="${ROOT_DIR}/boot/efi"
 
 cleanup() {
@@ -58,7 +69,7 @@ cleanup() {
 }
 
 rm -f "${OS_IMAGE}"
-qemu-img create -f qcow2 "${TARGET_OS_DIR}/krata.qcow2" "2G"
+qemu-img create -f qcow2 "${OS_IMAGE}" "2G"
 
 trap cleanup EXIT HUP INT TERM
 sudo qemu-nbd --connect="${NBD_DEVICE}" --cache=writeback -f qcow2 "${OS_IMAGE}"
@@ -75,7 +86,7 @@ sudo mount -t ext4 "${ROOT_PART}" "${ROOT_DIR}"
 sudo mkdir -p "${EFI_DIR}"
 sudo mount -t vfat "${EFI_PART}" "${EFI_DIR}"
 
-sudo tar xf "${TARGET_OS_DIR}/rootfs.tar" -C "${ROOT_DIR}"
+sudo tar xf "${ROOTFS}" -C "${ROOT_DIR}"
 ROOT_UUID="$(sudo blkid "${ROOT_PART}" | sed -En 's/.*\bUUID="([^"]+)".*/\1/p')"
 EFI_UUID="$(sudo blkid "${EFI_PART}" | sed -En 's/.*\bUUID="([^"]+)".*/\1/p')"
 echo "${ROOT_UUID}"
@@ -89,7 +100,7 @@ sudo mount --make-private "${ROOT_DIR}/sys"
 
 sudo cp "${PWD}/os/internal/stage2.sh" "${ROOT_DIR}/stage2.sh"
 echo "${ROOT_UUID}" | sudo tee "${ROOT_DIR}/root-uuid" > /dev/null
-sudo chroot "${ROOT_DIR}" /bin/sh -c "/stage2.sh"
+sudo chroot "${ROOT_DIR}" /bin/sh -c "/stage2.sh ${TARGET_ARCH} ${TARGET_ARCH_ALT}"
 sudo rm -f "${ROOT_DIR}/stage2.sh"
 sudo rm -f "${ROOT_DIR}/root-uuid"
 
@@ -101,6 +112,6 @@ sudo rm -f "${ROOT_DIR}/root-uuid"
 
 cleanup
 
-OS_SMALL_IMAGE="${TARGET_OS_DIR}/krata.small.qcow2"
+OS_SMALL_IMAGE="${TARGET_OS_DIR}/krata-${TARGET_ARCH}.small.qcow2"
 qemu-img convert -O qcow2 "${OS_IMAGE}" "${OS_SMALL_IMAGE}"
 mv -f "${OS_SMALL_IMAGE}" "${OS_IMAGE}"