From a34de3739a1c9992eb31aabc9fc504cf3a9f776a Mon Sep 17 00:00:00 2001 From: Ingolf Wagner Date: Mon, 27 Dec 2021 11:18:48 +0100 Subject: [PATCH] add hetzner dedicated install script --- ...etzner-dedicated-wipe-and-install-nixos.sh | 435 ++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100644 scripts/hetzner-dedicated-wipe-and-install-nixos.sh diff --git a/scripts/hetzner-dedicated-wipe-and-install-nixos.sh b/scripts/hetzner-dedicated-wipe-and-install-nixos.sh new file mode 100644 index 0000000..31560a6 --- /dev/null +++ b/scripts/hetzner-dedicated-wipe-and-install-nixos.sh @@ -0,0 +1,435 @@ +#!/usr/bin/env bash + +# Installs NixOS on a Hetzner server, wiping the server. +# +# This is for a specific server configuration; adjust where needed. +# +# Prerequisites: +# * Update the script wherever FIXME is present +# +# Usage: +# ssh root@YOUR_SERVERS_IP bash -s < hetzner-dedicated-wipe-and-install-nixos.sh +# +# When the script is done, make sure to boot the server from HD, not rescue mode again. + +# Explanations: +# +# * Adapted from https://gist.github.com/nh2/78d1c65e33806e7728622dbe748c2b6a +# * Following largely https://nixos.org/nixos/manual/index.html#sec-installing-from-other-distro. +# * **Important:** We boot in legacy-BIOS mode, not UEFI, because that's what Hetzner uses. +# * NVMe devices aren't supported for booting (those require EFI boot) +# * We set a custom `configuration.nix` so that we can connect to the machine afterwards, +# inspired by https://nixos.wiki/wiki/Install_NixOS_on_Hetzner_Online +# * This server has 2 HDDs. +# We put everything on RAID1. +# Storage scheme: `partitions -> RAID -> LVM -> ext4`. +# * A root user with empty password is created, so that you can just login +# as root and press enter when using the Hetzner spider KVM. +# Of course that empty-password login isn't exposed to the Internet. +# Change the password afterwards to avoid anyone with physical access +# being able to login without any authentication. +# * The script reboots at the end. + +# * does not use uefi (check if you can : efibootmgr) +# Notes https://mazzo.li/posts/hetzner-zfs.html + +# FIXME : change password +MAIN_PASSWORD="KlEBgwLgksT71cfIixM3eNDjIaZgFFvMDY8EoBs1Il" + +set -eu +set -o pipefail + +set -x + +# Inspect existing disks +lsblk + +# Undo existing setups to allow running the script multiple times to iterate on it. +# We allow these operations to fail for the case the script runs the first time. +set +e +umount /mnt/boot-{1,2} +umount /mnt +vgchange -an +cryptsetup close a_encrypted +cryptsetup close b_encrypted +set -e + + +# Stop all mdadm arrays that the boot may have activated. +mdadm --stop --scan + +# Prevent mdadm from auto-assembling arrays. +# Otherwise, as soon as we create the partition tables below, it will try to +# re-assemple a previous RAID if any remaining RAID signatures are present, +# before we even get the chance to wipe them. +# From: +# https://unix.stackexchange.com/questions/166688/prevent-debian-from-auto-assembling-raid-at-boot/504035#504035 +# We use `>` because the file may already contain some detected RAID arrays, +# which would take precedence over our ``. +echo 'AUTO -all +ARRAY UUID=00000000:00000000:00000000:00000000' > /etc/mdadm/mdadm.conf + +# Create partition tables (--script to not ask) +#parted --script /dev/sda mklabel gpt +#parted --script /dev/sdb mklabel gpt + +#parted /dev/sda -- mklabel gpt +#parted /dev/sda -- mkpart ESP fat32 1MiB 512MiB +#parted /dev/sda -- set 1 boot on +#parted /dev/sda -- mkpart primary 512MiB 100% + +format() { + parted -s "$1" -- mklabel gpt + + parted -s "$1" -- mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on + #parted -s "$1" -- mkpart 'boot' 2MB 512MiB + #parted -s "$1" -- mkpart ESP fat32 2MB 512MiB + parted -s "$1" -- mkpart ESP fat32 2MB 512MiB set 2 boot on + + #parted -s "$1" -- mkpart ESP fat32 1MiB 512MiB + + parted -s "$1" -- mkpart primary 512MiB 100% + parted -s "$1" -- print +} + +# In this particular machine we have two NVMe disks +format /dev/sda +format /dev/sdb + + +# Create partitions (--script to not ask) +# +# We create the 1MB BIOS boot partition at the front. +# +# Note we use "MB" instead of "MiB" because otherwise `--align optimal` has no effect; +# as per documentation https://www.gnu.org/software/parted/manual/html_node/unit.html#unit: +# > Note that as of parted-2.4, when you specify start and/or end values using IEC +# > binary units like "MiB", "GiB", "TiB", etc., parted treats those values as exact +# +# Note: When using `mkpart` on GPT, as per +# https://www.gnu.org/software/parted/manual/html_node/mkpart.html#mkpart +# the first argument to `mkpart` is not a `part-type`, but the GPT partition name: +# ... part-type is one of 'primary', 'extended' or 'logical', and may be specified only with 'msdos' or 'dvh' partition tables. +# A name must be specified for a 'gpt' partition table. +# GPT partition names are limited to 36 UTF-16 chars, see https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_entries_(LBA_2-33). +#parted --script --align optimal /dev/sda -- mklabel gpt mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on mkpart 'data-partition' 2MB '100%' +#parted --script --align optimal /dev/sdb -- mklabel gpt mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on mkpart 'data-partition' 2MB '100%' + +# Relaod partitions +#partprobe + +# Wait for all devices to exist +udevadm settle --timeout=5 --exit-if-exists=/dev/sda1 +udevadm settle --timeout=5 --exit-if-exists=/dev/sda2 +udevadm settle --timeout=5 --exit-if-exists=/dev/sda3 +udevadm settle --timeout=5 --exit-if-exists=/dev/sdb1 +udevadm settle --timeout=5 --exit-if-exists=/dev/sdb2 +udevadm settle --timeout=5 --exit-if-exists=/dev/sdb3 + +# Wipe any previous RAID signatures +#mdadm --zero-superblock --force /dev/sda2 +#mdadm --zero-superblock --force /dev/sdb2 + +# Create RAIDs +# Note that during creating and boot-time assembly, mdadm cares about the +# host name, and the existence and contents of `mdadm.conf`! +# This also affects the names appearing in /dev/md/ being different +# before and after reboot in general (but we take extra care here +# to pass explicit names, and set HOMEHOST for the rebooting system further +# down, so that the names appear the same). +# Almost all details of this are explained in +# https://bugzilla.redhat.com/show_bug.cgi?id=606481#c14 +# and the followup comments by Doug Ledford. +#mdadm --create --run --verbose /dev/md0 --level=1 --raid-devices=2 --homehost=hetzner --name=root0 /dev/sda2 /dev/sdb2 + +# Assembling the RAID can result in auto-activation of previously-existing LVM +# groups, preventing the RAID block device wiping below with +# `Device or resource busy`. So disable all VGs first. +#vgchange -an + +# Wipe filesystem signatures that might be on the RAID from some +# possibly existing older use of the disks (RAID creation does not do that). +# See https://serverfault.com/questions/911370/why-does-mdadm-zero-superblock-preserve-file-system-information +#wipefs -a /dev/md0 + +# Disable RAID recovery. We don't want this to slow down machine provisioning +# in the rescue mode. It can run in normal operation after reboot. +echo 0 > /proc/sys/dev/raid/speed_limit_max + +# LVM + +encrypt() { + device=$1 + label=$2 + echo $MAIN_PASSWORD | cryptsetup luksFormat ${device}3 - + echo $MAIN_PASSWORD | cryptsetup --key-file - open --type luks ${device}3 ${label}_encrypted +} + +encrypt /dev/sda "a" +encrypt /dev/sdb "b" + + + +# PVs +#pvcreate /dev/md0 + +pvcreate /dev/mapper/a_encrypted +pvcreate /dev/mapper/b_encrypted + +# VGs +vgcreate vg /dev/mapper/a_encrypted /dev/mapper/b_encrypted +#vgcreate vg0 /dev/md0 +# LVs (--yes to automatically wipe detected file system signatures) + +# the root partition should be raid1 +#lvcreate --mirrors 1 --type raid1 -L 150G --nosync -n root vg +lvcreate --mirrors 1 --type raid1 -L 150G -n root vg + +# Filesystems (-F to not ask on preexisting FS) +mkfs.ext4 -F -L root /dev/mapper/vg-root + +#mkfs.ext4 -F -L boot /dev/sda2 +mkfs.fat -F 32 -n boot /dev/sda2 +#mkfs.vfat -n boot /dev/sda2 + +#mkfs.ext4 -F -L boot /dev/sdb2 +mkfs.fat -F 32 -n boot /dev/sdb2 +#mkfs.vfat -n boot /dev/sdb2 + +# Creating file systems changes their UUIDs. +# Trigger udev so that the entries in /dev/disk/by-uuid get refreshed. +# `nixos-generate-config` depends on those being up-to-date. +# See https://github.com/NixOS/nixpkgs/issues/62444 +udevadm trigger + +# Wait for FS labels to appear +udevadm settle --timeout=5 --exit-if-exists=/dev/disk/by-label/root + +# NixOS pre-installation mounts + +# Mount target root partition +mount /dev/disk/by-label/root /mnt +mkdir -p /mnt/boot-{1,2} +modprobe vfat +mount /dev/sda2 /mnt/boot-1 +mount /dev/sdb2 /mnt/boot-2 + +# Installing nix + +# Installing nix requires `sudo`; the Hetzner rescue mode doesn't have it. +apt-get install -y sudo + +# Allow installing nix as root, see +# https://github.com/NixOS/nix/issues/936#issuecomment-475795730 +mkdir -p /etc/nix +echo "build-users-group =" > /etc/nix/nix.conf + +curl -L https://nixos.org/nix/install | sh +set +u +x # sourcing this may refer to unset variables that we have no control over +. $HOME/.nix-profile/etc/profile.d/nix.sh +set -u -x + +# FIXME Keep in sync with `system.stateVersion` set below! +nix-channel --add https://nixos.org/channels/nixos-21.05 nixpkgs +nix-channel --update + +# Getting NixOS installation tools +nix-env -iE "_: with import { configuration = {}; }; with config.system.build; [ nixos-generate-config nixos-install nixos-enter manual.manpages ]" + +nixos-generate-config --root /mnt + +# Find the name of the network interface that connects us to the Internet. +# Inspired by https://unix.stackexchange.com/questions/14961/how-to-find-out-which-interface-am-i-using-for-connecting-to-the-internet/302613#302613 +RESCUE_INTERFACE=$(ip route get 8.8.8.8 | grep -Po '(?<=dev )(\S+)') + +# Find what its name will be under NixOS, which uses stable interface names. +# See https://major.io/2015/08/21/understanding-systemds-predictable-network-device-names/#comment-545626 +# NICs for most Hetzner servers are not onboard, which is why we use +# `ID_NET_NAME_PATH`otherwise it would be `ID_NET_NAME_ONBOARD`. +INTERFACE_DEVICE_PATH=$(udevadm info -e | grep -Po "(?<=^P: )(.*${RESCUE_INTERFACE})") +UDEVADM_PROPERTIES_FOR_INTERFACE=$(udevadm info --query=property "--path=$INTERFACE_DEVICE_PATH") +NIXOS_INTERFACE=$(echo "$UDEVADM_PROPERTIES_FOR_INTERFACE" | grep -o -E 'ID_NET_NAME_PATH=\w+' | cut -d= -f2) +echo "Determined NIXOS_INTERFACE as '$NIXOS_INTERFACE'" +#NIXOS_INTERFACE='eth0' + +IP_V4=$(ip route get 8.8.8.8 | grep -Po '(?<=src )(\S+)') +echo "Determined IP_V4 as $IP_V4" + +# Determine Internet IPv6 by checking route, and using ::1 +# (because Hetzner rescue mode uses ::2 by default). +# The `ip -6 route get` output on Hetzner looks like: +# # ip -6 route get 2001:4860:4860:0:0:0:0:8888 +# 2001:4860:4860::8888 via fe80::1 dev eth0 src 2a01:4f8:151:62aa::2 metric 1024 pref medium +IP_V6="$(ip route get 2001:4860:4860:0:0:0:0:8888 | head -1 | cut -d' ' -f7 | cut -d: -f1-4)::1" +echo "Determined IP_V6 as $IP_V6" + + +# From https://stackoverflow.com/questions/1204629/how-do-i-get-the-default-gateway-in-linux-given-the-destination/15973156#15973156 +read _ _ DEFAULT_GATEWAY _ < <(ip route list match 0/0); echo "$DEFAULT_GATEWAY" +echo "Determined DEFAULT_GATEWAY as $DEFAULT_GATEWAY" + + +# Generate `configuration.nix`. Note that we splice in shell variables. +cat > /mnt/etc/nixos/configuration.nix < + }; + ipv6 = { + address = "$IP_V6"; # the ipv6 addres + gateway = "fe80::1"; # the ipv6 gateway + prefixLength = 64; # shown in the control panel + }; + +in +{ + imports = + [ # Include the results of the hardware scan. + # ./hardware-configuration.nix + + # hardware-configuration content + { + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "ahci" "sd_mod" ]; + boot.initrd.kernelModules = [ "dm-snapshot" ]; + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModulePackages = [ ]; + + swapDevices = [ ]; + powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand"; + + } + ]; + + # Use GRUB2 as the boot loader. + # We don't use systemd-boot because Hetzner uses BIOS legacy boot. + boot.loader.systemd-boot.enable = false; + boot.loader.grub = { + enable = true; + efiSupport = false; + version = 2; + }; + + # This will mirror all UEFI files, kernels, grub menus and + # things needed to boot to the other drive. + boot.loader.grub.mirroredBoots = [ + { path = "/boot-1"; devices = [ "/dev/sda" ]; } + { path = "/boot-2"; devices = [ "/dev/sdb" ]; } + ]; + + # add later + # fileSystems."/boot-1".options = [ "nofail" ]; + # fileSystems."/boot-2".options = [ "nofail" ]; + + boot.initrd.luks.devices = { + a_encrypted = { + device = "/dev/sda3"; + preLVM = true; + }; + b_encrypted = { + device = "/dev/sdb3"; + preLVM = true; + }; + }; + + # root + # ---- + fileSystems."/" = { + options = [ "noatime" "nodiratime" "discard" ]; + device = "/dev/vg/root"; + fsType = "ext4"; + }; + + networking.hostName = hostName; + + # Network configuration (Hetzner uses static IP assignments, and we don't use DHCP here) + networking.useDHCP = false; + networking.interfaces.\${networkInterface} = { + ipv4 = { addresses = [{ address = ipv4.address; prefixLength = ipv4.prefixLength; }]; }; + ipv6 = { addresses = [{ address = ipv6.address; prefixLength = ipv6.prefixLength; }]; }; + }; + networking.defaultGateway = ipv4.gateway; + networking.defaultGateway6 = { address = ipv6.gateway; interface = networkInterface; }; + networking.nameservers = [ "8.8.8.8" ]; + + # Initial empty root password for easy login: + users.users.root.initialHashedPassword = ""; + services.openssh.permitRootLogin = "prohibit-password"; + + users.users.root.openssh.authorizedKeys.keys = [ + # FIXME : add ssh key + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC6uza62+Go9sBFs3XZE2OkugBv9PJ7Yv8ebCskE5WYPcahMZIKkQw+zkGI8EGzOPJhQEv2xk+XBf2VOzj0Fto4nh8X5+Llb1nM+YxQPk1SVlwbNAlhh24L1w2vKtBtMy277MF4EP+caGceYP6gki5+DzlPUSdFSAEFFWgN1WPkiyUii15Xi3QuCMR8F18dbwVUYbT11vwNhdiAXWphrQG+yPguALBGR+21JM6fffOln3BhoDUp2poVc5Qe2EBuUbRUV3/fOU4HwWVKZ7KCFvLZBSVFutXCj5HuNWJ5T3RuuxJSmY5lYuFZx9gD+n+DAEJt30iXWcaJlmUqQB5awcB1S2d9pJ141V4vjiCMKUJHIdspFrI23rFNYD9k2ZXDA8VOnQE33BzmgF9xOVh6qr4G0oEpsNqJoKybVTUeSyl4+ifzdQANouvySgLJV/pcqaxX1srSDIUlcM2vDMWAs3ryCa0aAlmAVZIHgRhh6wa+IXW8gIYt+5biPWUuihJ4zGBEwkyVXXf2xsecMWCAGPWPDL0/fBfY9krNfC5M2sqxey2ShFIq+R/wMdaI7yVjUCF2QIUNiIdFbJL6bDrDyHnEXJJN+rAo23jUoTZZRv7Jq3DB/A5H7a73VCcblZyUmwMSlpg3wos7pdw5Ctta3zQPoxoAKGS1uZ+yTeZbPMmdbw==" + ]; + + services.openssh.enable = true; + + system.stateVersion = "21.05"; + + # enable ssh on init + # ------------------ + boot.kernelParams = [ + # See for docs on this + # ip=::::::::: + # The server ip refers to the NFS server -- we don't need it. + "ip=\${ipv4.address}::\${ipv4.gateway}:\${ipv4.netmask}:\${hostName}-initrd:\${networkInterface}:off:8.8.8.8" + ]; + boot.initrd.availableKernelModules = [ networkInterfaceModule ]; + boot.initrd.network.enable = true; + boot.initrd.network.ssh = { + enable = true; + authorizedKeys = config.users.users.root.openssh.authorizedKeys.keys; + port = 22; + hostKeys = [ + /etc/secrets/initrd/ssh_host_rsa_key + /etc/secrets/initrd/ssh_host_ed25519_key + ]; + + }; + + + # make sure ip address is set after in initrd + #boot.initrd.network.postCommands = '' + # up ip addr add $IP_V4/32 dev eth0 + # #ip address add $IP_V4/32 dev eth0 + # #ip link set eth0 up + # #ip address add $IP_V4/32 dev $NIXOS_INTERFACE + # #ip link set eth0 up + #''; + + +} +EOF + + +mkdir -p /mnt/etc/secrets/initrd/ + +ssh-keygen -t rsa -N "" -f /mnt/etc/secrets/initrd/ssh_host_rsa_key +ssh-keygen -t ed25519 -N "" -f /mnt/etc/secrets/initrd/ssh_host_ed25519_key + + +# Install NixOS +#PATH="$PATH" NIX_PATH="$NIX_PATH" `which nixos-install` --no-root-passwd --root /mnt --max-jobs 40 +PATH="$PATH" `which nixos-install` --no-root-passwd --root /mnt --max-jobs 40 + +umount /mnt/boot-{1,2} +umount /mnt + +echo "Determined NIXOS_INTERFACE as '$NIXOS_INTERFACE'" +echo "Determined IP_V4 as $IP_V4" +echo "Determined IP_V6 as $IP_V6" +echo "Determined DEFAULT_GATEWAY as $DEFAULT_GATEWAY" +reboot