nixos-config/scripts/hetzner-dedicated-wipe-and-install-nixos.sh

#!/usr/bin/env bash

# Adepted from :
# * https://mazzo.li/posts/hetzner-zfs.html
# * https://nixos.wiki/wiki/Install_NixOS_on_Hetzner_Online

# Installs NixOS on a Hetzner server, wiping the server.
#
# This is for a specific server configuration; adjust where needed.
#
# Prerequisites:
#   * Update the script wherever FIXME is present
#
# Usage:
#     ssh root@YOUR_SERVERS_IP bash -s < hetzner-dedicated-wipe-and-install-nixos.sh
#
# When the script is done, make sure to boot the server from HD, not rescue mode again.

# Explanations:
#
# * Adapted from https://gist.github.com/nh2/78d1c65e33806e7728622dbe748c2b6a
# * Following largely https://nixos.org/nixos/manual/index.html#sec-installing-from-other-distro.
# * **Important:** We boot in legacy-BIOS mode, not UEFI, because that's what Hetzner uses.
#   * NVMe devices aren't supported for booting (those require EFI boot)
# * We set a custom `configuration.nix` so that we can connect to the machine afterwards,
#   inspired by https://nixos.wiki/wiki/Install_NixOS_on_Hetzner_Online
# * This server has 2 HDDs.
#   We encrypt all hard drives and put an LVM on it the main logical volume is mirrored
#   Storage scheme: `partitions -> dm-crypt -> LVM -> ext4`.
#
#   ┌────────┐                      ┌────────┐
#   │  sda1  │   ◄──── grub  ─────► │  sdb1  │
#   ├────────┤                      ├────────┤
#   │  sda2  │   ◄──── /boot ─────► │  sdb2  │
#   ├────────┤                      ├────────┤
#   │  sda3  ├───┐              ┌───┤  sdb3  │
#   └────────┘   │              │   └────────┘
#                │              │
#   ┌────────────▼┐            ┌▼────────────┐
#   │ a_encrypted │            │ b_encrypted │
#   └────────┬────┘            └────┬────────┘
#            │     ┌──────────┐     │
#            └─────►  LVM vg  ◄─────┘
#                  └────┬─────┘
#                       │
#         ┌─────────────▼──────────────┐
#         │ root (mirrored/raid1)      │
#         ├────────────────────────────┤
#         │ nextcloud (mirrored/raid1) │
#         ├────────────────────────────┤
#         │ media (raid0)              │
#         └────────────────────────────┘
# * A root user with empty password is created, so that you can just login
#   as root and press enter when using the Hetzner spider KVM.
#   Of course that empty-password login isn't exposed to the Internet.
#   Change the password afterwards to avoid anyone with physical access
#   being able to login without any authentication.
# * The script reboots at the end.
# * does not use uefi (check if you can : efibootmgr)
#

# FIXME : change password
MAIN_PASSWORD="YOLO"

set -eu
set -o pipefail

set -x

# Inspect existing disks
lsblk

# Cleanup
# -------

# Undo existing setups to allow running the script multiple times to iterate on it.
# We allow these operations to fail for the case the script runs the first time.
set +e
umount /mnt/boot-{1,2}
umount /mnt
vgchange -an
cryptsetup close a_encrypted
cryptsetup close b_encrypted
set -e

# Stop all mdadm arrays that the boot may have activated.
mdadm --stop --scan

# Create partition tables (--script to not ask)
# ---------------------------------------------

format() {
  parted --script "$1" -- mklabel gpt

  parted --script "$1" -- mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on
  #parted --script "$1" -- mkpart 'boot' 2MB 512MiB
  #parted --script "$1" -- mkpart ESP fat32 2MB 512MiB
  parted --script "$1" -- mkpart 'boot' fat32 2MB 512MiB set 2 boot on


  parted --script "$1" -- mkpart primary 512MiB 100%
  parted --script "$1" -- print
}

# In this particular machine we have two NVMe disks
format /dev/sda
format /dev/sdb


# Relaod partitions
partprobe

# Wait for all devices to exist
udevadm settle --timeout=5 --exit-if-exists=/dev/sda1
udevadm settle --timeout=5 --exit-if-exists=/dev/sda2
udevadm settle --timeout=5 --exit-if-exists=/dev/sda3
udevadm settle --timeout=5 --exit-if-exists=/dev/sdb1
udevadm settle --timeout=5 --exit-if-exists=/dev/sdb2
udevadm settle --timeout=5 --exit-if-exists=/dev/sdb3


# Disable RAID recovery. We don't want this to slow down machine provisioning
# in the rescue mode. It can run in normal operation after reboot.
echo 0 > /proc/sys/dev/raid/speed_limit_max

# LVM

encrypt() {
  device=$1
  label=$2
  echo -n "$MAIN_PASSWORD" | cryptsetup luksFormat ${device}3 -
  echo -n "$MAIN_PASSWORD" | cryptsetup --key-file - open --type luks ${device}3 ${label}_encrypted
}

encrypt /dev/sda "a"
encrypt /dev/sdb "b"



# /boot partitions
# ----------------


mkfs.fat -F 32 -n boot /dev/sda2
mkfs.fat -F 32 -n boot /dev/sdb2



# PVs

pvcreate /dev/mapper/a_encrypted
pvcreate /dev/mapper/b_encrypted

# VGs
vgcreate vg /dev/mapper/a_encrypted /dev/mapper/b_encrypted

# LVs (--yes to automatically wipe detected file system signatures)

# the root partition should be raid1
lvcreate --mirrors 1 --type raid1 -L 150G -n root vg

# Filesystems (-F to not ask on preexisting FS)
mkfs.ext4 -F -L root /dev/mapper/vg-root

# Creating file systems changes their UUIDs.
# Trigger udev so that the entries in /dev/disk/by-uuid get refreshed.
# `nixos-generate-config` depends on those being up-to-date.
# See https://github.com/NixOS/nixpkgs/issues/62444
udevadm trigger

# Wait for FS labels to appear
udevadm settle --timeout=5 --exit-if-exists=/dev/disk/by-label/root

# NixOS pre-installation mounts
# -----------------------------

# Mount target root partition
mount /dev/disk/by-label/root /mnt
mkdir -p /mnt/boot-{1,2}
modprobe vfat
mount /dev/sda2 /mnt/boot-1
mount /dev/sdb2 /mnt/boot-2

# Installing nix
# --------------

# Installing nix requires `sudo`; the Hetzner rescue mode doesn't have it.
apt-get install -y sudo

# Allow installing nix as root, see
#   https://github.com/NixOS/nix/issues/936#issuecomment-475795730
mkdir -p /etc/nix
echo "build-users-group =" > /etc/nix/nix.conf

curl -L https://nixos.org/nix/install | sh
set +u +x # sourcing this may refer to unset variables that we have no control over
. $HOME/.nix-profile/etc/profile.d/nix.sh
set -u -x

# FIXME Keep in sync with `system.stateVersion` set below!
nix-channel --add https://nixos.org/channels/nixos-21.05 nixpkgs
nix-channel --update

# Getting NixOS installation tools
nix-env -iE "_: with import <nixpkgs/nixos> { configuration = {}; }; with config.system.build; [ nixos-generate-config nixos-install nixos-enter manual.manpages ]"

nixos-generate-config --root /mnt

# Detect
# ------

# Find the name of the network interface that connects us to the Internet.
# Inspired by https://unix.stackexchange.com/questions/14961/how-to-find-out-which-interface-am-i-using-for-connecting-to-the-internet/302613#302613
RESCUE_INTERFACE=$(ip route get 8.8.8.8 | grep -Po '(?<=dev )(\S+)')

# Find what its name will be under NixOS, which uses stable interface names.
# See https://major.io/2015/08/21/understanding-systemds-predictable-network-device-names/#comment-545626
# NICs for most Hetzner servers are not onboard, which is why we use
# `ID_NET_NAME_PATH`otherwise it would be `ID_NET_NAME_ONBOARD`.
INTERFACE_DEVICE_PATH=$(udevadm info -e | grep -Po "(?<=^P: )(.*${RESCUE_INTERFACE})")
UDEVADM_PROPERTIES_FOR_INTERFACE=$(udevadm info --query=property "--path=$INTERFACE_DEVICE_PATH")
NIXOS_INTERFACE=$(echo "$UDEVADM_PROPERTIES_FOR_INTERFACE" | grep -o -E 'ID_NET_NAME_PATH=\w+' | cut -d= -f2)
echo "Determined NIXOS_INTERFACE as '$NIXOS_INTERFACE'"
#NIXOS_INTERFACE='eth0'

IP_V4=$(ip route get 8.8.8.8 | grep -Po '(?<=src )(\S+)')
echo "Determined IP_V4 as $IP_V4"

# Determine Internet IPv6 by checking route, and using ::1
# (because Hetzner rescue mode uses ::2 by default).
# The `ip -6 route get` output on Hetzner looks like:
#   # ip -6 route get 2001:4860:4860:0:0:0:0:8888
#   2001:4860:4860::8888 via fe80::1 dev eth0 src 2a01:4f8:151:62aa::2 metric 1024  pref medium
IP_V6="$(ip route get 2001:4860:4860:0:0:0:0:8888 | head -1 | cut -d' ' -f7 | cut -d: -f1-4)::1"
echo "Determined IP_V6 as $IP_V6"


# From https://stackoverflow.com/questions/1204629/how-do-i-get-the-default-gateway-in-linux-given-the-destination/15973156#15973156
read _ _ DEFAULT_GATEWAY _ < <(ip route list match 0/0); echo "$DEFAULT_GATEWAY"
echo "Determined DEFAULT_GATEWAY as $DEFAULT_GATEWAY"


# Generate `configuration.nix`. Note that we splice in shell variables.
cat > /mnt/etc/nixos/configuration.nix <<EOF
{ config, pkgs, modulesPath, lib, ... }:
let
  hostName = "hetzner";
  # FIXME configure network driver
  # in rescue
  # apt install -y lshw
  # lshw -C network | grep -Poh 'driver=[[:alnum:]]+'
  networkInterfaceModule = "r8169";
  networkInterface = "$NIXOS_INTERFACE";
  # From the Hetzner control panel
  ipv4 = {
    address = "$IP_V4";           # the ip address
    gateway = "$DEFAULT_GATEWAY"; # the gateway ip address
    netmask = "255.255.255.224";  # the netmask -- might not be the same for you!
    prefixLength = 27;            # must match the netmask, see <https://www.pawprint.net/designresources/netmask-converter.php>
  };
  ipv6 = {
    address = "$IP_V6";  # the ipv6 addres
    gateway = "fe80::1"; # the ipv6 gateway
    prefixLength = 64;   # shown in the control panel
  };
in
{
  imports =
    [ # Include the results of the hardware scan.
      ./hardware-configuration.nix
    ];

  # needed lvm for raid
  boot.initrd.kernelModules = [
    "dm-snapshot"
    "dm_mirror"
    "dm_raid"
    "dm_region_hash"
  ];

  # Use GRUB2 as the boot loader.
  # We don't use systemd-boot because Hetzner uses BIOS legacy boot.
  boot.loader.systemd-boot.enable = false;
  boot.loader.grub = {
    enable = true;
    efiSupport = false;
    version = 2;
  };

  # This will mirror all UEFI files, kernels, grub menus and
  # things needed to boot to the other drive.
  boot.loader.grub.mirroredBoots = [
    { path = "/boot-1"; devices = [ "/dev/sda" ]; }
    { path = "/boot-2"; devices = [ "/dev/sdb" ]; }
  ];

  # We want to still be able to boot without one of these
  fileSystems."/boot-1".options = [ "nofail" ];
  fileSystems."/boot-2".options = [ "nofail" ];

  boot.initrd.luks.reusePassphrases = true;
  boot.initrd.luks.devices = {
    a_encrypted = {
      device = "/dev/sda3";
      preLVM = true;
    };
    b_encrypted = {
      device = "/dev/sdb3";
      preLVM = true;
    };
  };

  networking.hostName = hostName;

  # Network configuration (Hetzner uses static IP assignments, and we don't use DHCP here)
  networking.useDHCP = false;
  networking.interfaces.\${networkInterface} = {
    ipv4 = { addresses = [{ address = ipv4.address; prefixLength = ipv4.prefixLength; }]; };
    ipv6 = { addresses = [{ address = ipv6.address; prefixLength = ipv6.prefixLength; }]; };
  };
  networking.defaultGateway = ipv4.gateway;
  networking.defaultGateway6 = { address = ipv6.gateway; interface = networkInterface; };
  networking.nameservers = [ "8.8.8.8" ];

  # Initial empty root password for easy login:
  users.users.root.initialHashedPassword = "";
  services.openssh.permitRootLogin = "prohibit-password";

  users.users.root.openssh.authorizedKeys.keys = [
    # FIXME : add ssh key
    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC6uza62+Go9sBFs3XZE2OkugBv9PJ7Yv8ebCskE5WYPcahMZIKkQw+zkGI8EGzOPJhQEv2xk+XBf2VOzj0Fto4nh8X5+Llb1nM+YxQPk1SVlwbNAlhh24L1w2vKtBtMy277MF4EP+caGceYP6gki5+DzlPUSdFSAEFFWgN1WPkiyUii15Xi3QuCMR8F18dbwVUYbT11vwNhdiAXWphrQG+yPguALBGR+21JM6fffOln3BhoDUp2poVc5Qe2EBuUbRUV3/fOU4HwWVKZ7KCFvLZBSVFutXCj5HuNWJ5T3RuuxJSmY5lYuFZx9gD+n+DAEJt30iXWcaJlmUqQB5awcB1S2d9pJ141V4vjiCMKUJHIdspFrI23rFNYD9k2ZXDA8VOnQE33BzmgF9xOVh6qr4G0oEpsNqJoKybVTUeSyl4+ifzdQANouvySgLJV/pcqaxX1srSDIUlcM2vDMWAs3ryCa0aAlmAVZIHgRhh6wa+IXW8gIYt+5biPWUuihJ4zGBEwkyVXXf2xsecMWCAGPWPDL0/fBfY9krNfC5M2sqxey2ShFIq+R/wMdaI7yVjUCF2QIUNiIdFbJL6bDrDyHnEXJJN+rAo23jUoTZZRv7Jq3DB/A5H7a73VCcblZyUmwMSlpg3wos7pdw5Ctta3zQPoxoAKGS1uZ+yTeZbPMmdbw=="
  ];

  services.openssh.enable = true;

  system.stateVersion = "21.05";

  # enable ssh on init
  # ------------------

  boot.kernelParams = [
    # See <https://www.kernel.org/doc/Documentation/filesystems/nfs/nfsroot.txt> for docs on this
    # ip=<client-ip>:<server-ip>:<gw-ip>:<netmask>:<hostname>:<device>:<autoconf>:<dns0-ip>:<dns1-ip>:<ntp0-ip>
    # The server ip refers to the NFS server -- we don't need it.
    "ip=\${ipv4.address}::\${ipv4.gateway}:\${ipv4.netmask}:\${hostName}-initrd:\${networkInterface}:off:8.8.8.8"
  ];
  boot.initrd.availableKernelModules = [ networkInterfaceModule ];
  boot.initrd.network.enable = true;
  boot.initrd.network.ssh = {
    enable = true;
    authorizedKeys = config.users.users.root.openssh.authorizedKeys.keys;
    port = 22;
    hostKeys = [
      /etc/secrets/initrd/ssh_host_rsa_key
      /etc/secrets/initrd/ssh_host_ed25519_key
    ];
  };

}
EOF


mkdir -p /mnt/etc/secrets/initrd/

ssh-keygen -t rsa -N "" -f /mnt/etc/secrets/initrd/ssh_host_rsa_key
ssh-keygen -t ed25519 -N "" -f /mnt/etc/secrets/initrd/ssh_host_ed25519_key


# Install NixOS
#PATH="$PATH" NIX_PATH="$NIX_PATH" `which nixos-install` --no-root-passwd --root /mnt --max-jobs 40
PATH="$PATH" `which nixos-install` --no-root-passwd --root /mnt --max-jobs 40

umount /mnt/boot-{1,2}
umount /mnt

echo "Determined NIXOS_INTERFACE as '$NIXOS_INTERFACE'"
echo "Determined IP_V4 as $IP_V4"
echo "Determined IP_V6 as $IP_V6"
echo "Determined DEFAULT_GATEWAY as $DEFAULT_GATEWAY"
reboot