435 lines
16 KiB
Bash
435 lines
16 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Installs NixOS on a Hetzner server, wiping the server.
|
|
#
|
|
# This is for a specific server configuration; adjust where needed.
|
|
#
|
|
# Prerequisites:
|
|
# * Update the script wherever FIXME is present
|
|
#
|
|
# Usage:
|
|
# ssh root@YOUR_SERVERS_IP bash -s < hetzner-dedicated-wipe-and-install-nixos.sh
|
|
#
|
|
# When the script is done, make sure to boot the server from HD, not rescue mode again.
|
|
|
|
# Explanations:
|
|
#
|
|
# * Adapted from https://gist.github.com/nh2/78d1c65e33806e7728622dbe748c2b6a
|
|
# * Following largely https://nixos.org/nixos/manual/index.html#sec-installing-from-other-distro.
|
|
# * **Important:** We boot in legacy-BIOS mode, not UEFI, because that's what Hetzner uses.
|
|
# * NVMe devices aren't supported for booting (those require EFI boot)
|
|
# * We set a custom `configuration.nix` so that we can connect to the machine afterwards,
|
|
# inspired by https://nixos.wiki/wiki/Install_NixOS_on_Hetzner_Online
|
|
# * This server has 2 HDDs.
|
|
# We put everything on RAID1.
|
|
# Storage scheme: `partitions -> RAID -> LVM -> ext4`.
|
|
# * A root user with empty password is created, so that you can just login
|
|
# as root and press enter when using the Hetzner spider KVM.
|
|
# Of course that empty-password login isn't exposed to the Internet.
|
|
# Change the password afterwards to avoid anyone with physical access
|
|
# being able to login without any authentication.
|
|
# * The script reboots at the end.
|
|
|
|
# * does not use uefi (check if you can : efibootmgr)
|
|
# Notes https://mazzo.li/posts/hetzner-zfs.html
|
|
|
|
# FIXME : change password
|
|
MAIN_PASSWORD="KlEBgwLgksT71cfIixM3eNDjIaZgFFvMDY8EoBs1Il"
|
|
|
|
set -eu
|
|
set -o pipefail
|
|
|
|
set -x
|
|
|
|
# Inspect existing disks
|
|
lsblk
|
|
|
|
# Undo existing setups to allow running the script multiple times to iterate on it.
|
|
# We allow these operations to fail for the case the script runs the first time.
|
|
set +e
|
|
umount /mnt/boot-{1,2}
|
|
umount /mnt
|
|
vgchange -an
|
|
cryptsetup close a_encrypted
|
|
cryptsetup close b_encrypted
|
|
set -e
|
|
|
|
|
|
# Stop all mdadm arrays that the boot may have activated.
|
|
mdadm --stop --scan
|
|
|
|
# Prevent mdadm from auto-assembling arrays.
|
|
# Otherwise, as soon as we create the partition tables below, it will try to
|
|
# re-assemple a previous RAID if any remaining RAID signatures are present,
|
|
# before we even get the chance to wipe them.
|
|
# From:
|
|
# https://unix.stackexchange.com/questions/166688/prevent-debian-from-auto-assembling-raid-at-boot/504035#504035
|
|
# We use `>` because the file may already contain some detected RAID arrays,
|
|
# which would take precedence over our `<ignore>`.
|
|
echo 'AUTO -all
|
|
ARRAY <ignore> UUID=00000000:00000000:00000000:00000000' > /etc/mdadm/mdadm.conf
|
|
|
|
# Create partition tables (--script to not ask)
|
|
#parted --script /dev/sda mklabel gpt
|
|
#parted --script /dev/sdb mklabel gpt
|
|
|
|
#parted /dev/sda -- mklabel gpt
|
|
#parted /dev/sda -- mkpart ESP fat32 1MiB 512MiB
|
|
#parted /dev/sda -- set 1 boot on
|
|
#parted /dev/sda -- mkpart primary 512MiB 100%
|
|
|
|
format() {
|
|
parted -s "$1" -- mklabel gpt
|
|
|
|
parted -s "$1" -- mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on
|
|
#parted -s "$1" -- mkpart 'boot' 2MB 512MiB
|
|
#parted -s "$1" -- mkpart ESP fat32 2MB 512MiB
|
|
parted -s "$1" -- mkpart ESP fat32 2MB 512MiB set 2 boot on
|
|
|
|
#parted -s "$1" -- mkpart ESP fat32 1MiB 512MiB
|
|
|
|
parted -s "$1" -- mkpart primary 512MiB 100%
|
|
parted -s "$1" -- print
|
|
}
|
|
|
|
# In this particular machine we have two NVMe disks
|
|
format /dev/sda
|
|
format /dev/sdb
|
|
|
|
|
|
# Create partitions (--script to not ask)
|
|
#
|
|
# We create the 1MB BIOS boot partition at the front.
|
|
#
|
|
# Note we use "MB" instead of "MiB" because otherwise `--align optimal` has no effect;
|
|
# as per documentation https://www.gnu.org/software/parted/manual/html_node/unit.html#unit:
|
|
# > Note that as of parted-2.4, when you specify start and/or end values using IEC
|
|
# > binary units like "MiB", "GiB", "TiB", etc., parted treats those values as exact
|
|
#
|
|
# Note: When using `mkpart` on GPT, as per
|
|
# https://www.gnu.org/software/parted/manual/html_node/mkpart.html#mkpart
|
|
# the first argument to `mkpart` is not a `part-type`, but the GPT partition name:
|
|
# ... part-type is one of 'primary', 'extended' or 'logical', and may be specified only with 'msdos' or 'dvh' partition tables.
|
|
# A name must be specified for a 'gpt' partition table.
|
|
# GPT partition names are limited to 36 UTF-16 chars, see https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_entries_(LBA_2-33).
|
|
#parted --script --align optimal /dev/sda -- mklabel gpt mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on mkpart 'data-partition' 2MB '100%'
|
|
#parted --script --align optimal /dev/sdb -- mklabel gpt mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on mkpart 'data-partition' 2MB '100%'
|
|
|
|
# Relaod partitions
|
|
#partprobe
|
|
|
|
# Wait for all devices to exist
|
|
udevadm settle --timeout=5 --exit-if-exists=/dev/sda1
|
|
udevadm settle --timeout=5 --exit-if-exists=/dev/sda2
|
|
udevadm settle --timeout=5 --exit-if-exists=/dev/sda3
|
|
udevadm settle --timeout=5 --exit-if-exists=/dev/sdb1
|
|
udevadm settle --timeout=5 --exit-if-exists=/dev/sdb2
|
|
udevadm settle --timeout=5 --exit-if-exists=/dev/sdb3
|
|
|
|
# Wipe any previous RAID signatures
|
|
#mdadm --zero-superblock --force /dev/sda2
|
|
#mdadm --zero-superblock --force /dev/sdb2
|
|
|
|
# Create RAIDs
|
|
# Note that during creating and boot-time assembly, mdadm cares about the
|
|
# host name, and the existence and contents of `mdadm.conf`!
|
|
# This also affects the names appearing in /dev/md/ being different
|
|
# before and after reboot in general (but we take extra care here
|
|
# to pass explicit names, and set HOMEHOST for the rebooting system further
|
|
# down, so that the names appear the same).
|
|
# Almost all details of this are explained in
|
|
# https://bugzilla.redhat.com/show_bug.cgi?id=606481#c14
|
|
# and the followup comments by Doug Ledford.
|
|
#mdadm --create --run --verbose /dev/md0 --level=1 --raid-devices=2 --homehost=hetzner --name=root0 /dev/sda2 /dev/sdb2
|
|
|
|
# Assembling the RAID can result in auto-activation of previously-existing LVM
|
|
# groups, preventing the RAID block device wiping below with
|
|
# `Device or resource busy`. So disable all VGs first.
|
|
#vgchange -an
|
|
|
|
# Wipe filesystem signatures that might be on the RAID from some
|
|
# possibly existing older use of the disks (RAID creation does not do that).
|
|
# See https://serverfault.com/questions/911370/why-does-mdadm-zero-superblock-preserve-file-system-information
|
|
#wipefs -a /dev/md0
|
|
|
|
# Disable RAID recovery. We don't want this to slow down machine provisioning
|
|
# in the rescue mode. It can run in normal operation after reboot.
|
|
echo 0 > /proc/sys/dev/raid/speed_limit_max
|
|
|
|
# LVM
|
|
|
|
encrypt() {
|
|
device=$1
|
|
label=$2
|
|
echo $MAIN_PASSWORD | cryptsetup luksFormat ${device}3 -
|
|
echo $MAIN_PASSWORD | cryptsetup --key-file - open --type luks ${device}3 ${label}_encrypted
|
|
}
|
|
|
|
encrypt /dev/sda "a"
|
|
encrypt /dev/sdb "b"
|
|
|
|
|
|
|
|
# PVs
|
|
#pvcreate /dev/md0
|
|
|
|
pvcreate /dev/mapper/a_encrypted
|
|
pvcreate /dev/mapper/b_encrypted
|
|
|
|
# VGs
|
|
vgcreate vg /dev/mapper/a_encrypted /dev/mapper/b_encrypted
|
|
#vgcreate vg0 /dev/md0
|
|
# LVs (--yes to automatically wipe detected file system signatures)
|
|
|
|
# the root partition should be raid1
|
|
#lvcreate --mirrors 1 --type raid1 -L 150G --nosync -n root vg
|
|
lvcreate --mirrors 1 --type raid1 -L 150G -n root vg
|
|
|
|
# Filesystems (-F to not ask on preexisting FS)
|
|
mkfs.ext4 -F -L root /dev/mapper/vg-root
|
|
|
|
#mkfs.ext4 -F -L boot /dev/sda2
|
|
mkfs.fat -F 32 -n boot /dev/sda2
|
|
#mkfs.vfat -n boot /dev/sda2
|
|
|
|
#mkfs.ext4 -F -L boot /dev/sdb2
|
|
mkfs.fat -F 32 -n boot /dev/sdb2
|
|
#mkfs.vfat -n boot /dev/sdb2
|
|
|
|
# Creating file systems changes their UUIDs.
|
|
# Trigger udev so that the entries in /dev/disk/by-uuid get refreshed.
|
|
# `nixos-generate-config` depends on those being up-to-date.
|
|
# See https://github.com/NixOS/nixpkgs/issues/62444
|
|
udevadm trigger
|
|
|
|
# Wait for FS labels to appear
|
|
udevadm settle --timeout=5 --exit-if-exists=/dev/disk/by-label/root
|
|
|
|
# NixOS pre-installation mounts
|
|
|
|
# Mount target root partition
|
|
mount /dev/disk/by-label/root /mnt
|
|
mkdir -p /mnt/boot-{1,2}
|
|
modprobe vfat
|
|
mount /dev/sda2 /mnt/boot-1
|
|
mount /dev/sdb2 /mnt/boot-2
|
|
|
|
# Installing nix
|
|
|
|
# Installing nix requires `sudo`; the Hetzner rescue mode doesn't have it.
|
|
apt-get install -y sudo
|
|
|
|
# Allow installing nix as root, see
|
|
# https://github.com/NixOS/nix/issues/936#issuecomment-475795730
|
|
mkdir -p /etc/nix
|
|
echo "build-users-group =" > /etc/nix/nix.conf
|
|
|
|
curl -L https://nixos.org/nix/install | sh
|
|
set +u +x # sourcing this may refer to unset variables that we have no control over
|
|
. $HOME/.nix-profile/etc/profile.d/nix.sh
|
|
set -u -x
|
|
|
|
# FIXME Keep in sync with `system.stateVersion` set below!
|
|
nix-channel --add https://nixos.org/channels/nixos-21.05 nixpkgs
|
|
nix-channel --update
|
|
|
|
# Getting NixOS installation tools
|
|
nix-env -iE "_: with import <nixpkgs/nixos> { configuration = {}; }; with config.system.build; [ nixos-generate-config nixos-install nixos-enter manual.manpages ]"
|
|
|
|
nixos-generate-config --root /mnt
|
|
|
|
# Find the name of the network interface that connects us to the Internet.
|
|
# Inspired by https://unix.stackexchange.com/questions/14961/how-to-find-out-which-interface-am-i-using-for-connecting-to-the-internet/302613#302613
|
|
RESCUE_INTERFACE=$(ip route get 8.8.8.8 | grep -Po '(?<=dev )(\S+)')
|
|
|
|
# Find what its name will be under NixOS, which uses stable interface names.
|
|
# See https://major.io/2015/08/21/understanding-systemds-predictable-network-device-names/#comment-545626
|
|
# NICs for most Hetzner servers are not onboard, which is why we use
|
|
# `ID_NET_NAME_PATH`otherwise it would be `ID_NET_NAME_ONBOARD`.
|
|
INTERFACE_DEVICE_PATH=$(udevadm info -e | grep -Po "(?<=^P: )(.*${RESCUE_INTERFACE})")
|
|
UDEVADM_PROPERTIES_FOR_INTERFACE=$(udevadm info --query=property "--path=$INTERFACE_DEVICE_PATH")
|
|
NIXOS_INTERFACE=$(echo "$UDEVADM_PROPERTIES_FOR_INTERFACE" | grep -o -E 'ID_NET_NAME_PATH=\w+' | cut -d= -f2)
|
|
echo "Determined NIXOS_INTERFACE as '$NIXOS_INTERFACE'"
|
|
#NIXOS_INTERFACE='eth0'
|
|
|
|
IP_V4=$(ip route get 8.8.8.8 | grep -Po '(?<=src )(\S+)')
|
|
echo "Determined IP_V4 as $IP_V4"
|
|
|
|
# Determine Internet IPv6 by checking route, and using ::1
|
|
# (because Hetzner rescue mode uses ::2 by default).
|
|
# The `ip -6 route get` output on Hetzner looks like:
|
|
# # ip -6 route get 2001:4860:4860:0:0:0:0:8888
|
|
# 2001:4860:4860::8888 via fe80::1 dev eth0 src 2a01:4f8:151:62aa::2 metric 1024 pref medium
|
|
IP_V6="$(ip route get 2001:4860:4860:0:0:0:0:8888 | head -1 | cut -d' ' -f7 | cut -d: -f1-4)::1"
|
|
echo "Determined IP_V6 as $IP_V6"
|
|
|
|
|
|
# From https://stackoverflow.com/questions/1204629/how-do-i-get-the-default-gateway-in-linux-given-the-destination/15973156#15973156
|
|
read _ _ DEFAULT_GATEWAY _ < <(ip route list match 0/0); echo "$DEFAULT_GATEWAY"
|
|
echo "Determined DEFAULT_GATEWAY as $DEFAULT_GATEWAY"
|
|
|
|
|
|
# Generate `configuration.nix`. Note that we splice in shell variables.
|
|
cat > /mnt/etc/nixos/configuration.nix <<EOF
|
|
{ config, pkgs, modulesPath, lib, ... }:
|
|
let
|
|
hostName = "hetzner";
|
|
# in rescue
|
|
# apt install -y lshw
|
|
# lshw -C network | grep -Poh 'driver=[[:alnum:]]+'
|
|
networkInterfaceModule = "r8169";
|
|
networkInterface = "$NIXOS_INTERFACE";
|
|
# From the Hetzner control panel
|
|
ipv4 = {
|
|
address = "$IP_V4"; # the ip address
|
|
gateway = "$DEFAULT_GATEWAY"; # the gateway ip address
|
|
netmask = "255.255.255.224"; # the netmask -- might not be the same for you!
|
|
prefixLength = 27; # must match the netmask, see <https://www.pawprint.net/designresources/netmask-converter.php>
|
|
};
|
|
ipv6 = {
|
|
address = "$IP_V6"; # the ipv6 addres
|
|
gateway = "fe80::1"; # the ipv6 gateway
|
|
prefixLength = 64; # shown in the control panel
|
|
};
|
|
|
|
in
|
|
{
|
|
imports =
|
|
[ # Include the results of the hardware scan.
|
|
# ./hardware-configuration.nix
|
|
|
|
# hardware-configuration content
|
|
{
|
|
imports =
|
|
[ (modulesPath + "/installer/scan/not-detected.nix")
|
|
];
|
|
|
|
boot.initrd.availableKernelModules = [ "ahci" "sd_mod" ];
|
|
boot.initrd.kernelModules = [ "dm-snapshot" ];
|
|
boot.kernelModules = [ "kvm-intel" ];
|
|
boot.extraModulePackages = [ ];
|
|
|
|
swapDevices = [ ];
|
|
powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
|
|
|
|
}
|
|
];
|
|
|
|
# Use GRUB2 as the boot loader.
|
|
# We don't use systemd-boot because Hetzner uses BIOS legacy boot.
|
|
boot.loader.systemd-boot.enable = false;
|
|
boot.loader.grub = {
|
|
enable = true;
|
|
efiSupport = false;
|
|
version = 2;
|
|
};
|
|
|
|
# This will mirror all UEFI files, kernels, grub menus and
|
|
# things needed to boot to the other drive.
|
|
boot.loader.grub.mirroredBoots = [
|
|
{ path = "/boot-1"; devices = [ "/dev/sda" ]; }
|
|
{ path = "/boot-2"; devices = [ "/dev/sdb" ]; }
|
|
];
|
|
|
|
# add later
|
|
# fileSystems."/boot-1".options = [ "nofail" ];
|
|
# fileSystems."/boot-2".options = [ "nofail" ];
|
|
|
|
boot.initrd.luks.devices = {
|
|
a_encrypted = {
|
|
device = "/dev/sda3";
|
|
preLVM = true;
|
|
};
|
|
b_encrypted = {
|
|
device = "/dev/sdb3";
|
|
preLVM = true;
|
|
};
|
|
};
|
|
|
|
# root
|
|
# ----
|
|
fileSystems."/" = {
|
|
options = [ "noatime" "nodiratime" "discard" ];
|
|
device = "/dev/vg/root";
|
|
fsType = "ext4";
|
|
};
|
|
|
|
networking.hostName = hostName;
|
|
|
|
# Network configuration (Hetzner uses static IP assignments, and we don't use DHCP here)
|
|
networking.useDHCP = false;
|
|
networking.interfaces.\${networkInterface} = {
|
|
ipv4 = { addresses = [{ address = ipv4.address; prefixLength = ipv4.prefixLength; }]; };
|
|
ipv6 = { addresses = [{ address = ipv6.address; prefixLength = ipv6.prefixLength; }]; };
|
|
};
|
|
networking.defaultGateway = ipv4.gateway;
|
|
networking.defaultGateway6 = { address = ipv6.gateway; interface = networkInterface; };
|
|
networking.nameservers = [ "8.8.8.8" ];
|
|
|
|
# Initial empty root password for easy login:
|
|
users.users.root.initialHashedPassword = "";
|
|
services.openssh.permitRootLogin = "prohibit-password";
|
|
|
|
users.users.root.openssh.authorizedKeys.keys = [
|
|
# FIXME : add ssh key
|
|
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC6uza62+Go9sBFs3XZE2OkugBv9PJ7Yv8ebCskE5WYPcahMZIKkQw+zkGI8EGzOPJhQEv2xk+XBf2VOzj0Fto4nh8X5+Llb1nM+YxQPk1SVlwbNAlhh24L1w2vKtBtMy277MF4EP+caGceYP6gki5+DzlPUSdFSAEFFWgN1WPkiyUii15Xi3QuCMR8F18dbwVUYbT11vwNhdiAXWphrQG+yPguALBGR+21JM6fffOln3BhoDUp2poVc5Qe2EBuUbRUV3/fOU4HwWVKZ7KCFvLZBSVFutXCj5HuNWJ5T3RuuxJSmY5lYuFZx9gD+n+DAEJt30iXWcaJlmUqQB5awcB1S2d9pJ141V4vjiCMKUJHIdspFrI23rFNYD9k2ZXDA8VOnQE33BzmgF9xOVh6qr4G0oEpsNqJoKybVTUeSyl4+ifzdQANouvySgLJV/pcqaxX1srSDIUlcM2vDMWAs3ryCa0aAlmAVZIHgRhh6wa+IXW8gIYt+5biPWUuihJ4zGBEwkyVXXf2xsecMWCAGPWPDL0/fBfY9krNfC5M2sqxey2ShFIq+R/wMdaI7yVjUCF2QIUNiIdFbJL6bDrDyHnEXJJN+rAo23jUoTZZRv7Jq3DB/A5H7a73VCcblZyUmwMSlpg3wos7pdw5Ctta3zQPoxoAKGS1uZ+yTeZbPMmdbw=="
|
|
];
|
|
|
|
services.openssh.enable = true;
|
|
|
|
system.stateVersion = "21.05";
|
|
|
|
# enable ssh on init
|
|
# ------------------
|
|
boot.kernelParams = [
|
|
# See <https://www.kernel.org/doc/Documentation/filesystems/nfs/nfsroot.txt> for docs on this
|
|
# ip=<client-ip>:<server-ip>:<gw-ip>:<netmask>:<hostname>:<device>:<autoconf>:<dns0-ip>:<dns1-ip>:<ntp0-ip>
|
|
# The server ip refers to the NFS server -- we don't need it.
|
|
"ip=\${ipv4.address}::\${ipv4.gateway}:\${ipv4.netmask}:\${hostName}-initrd:\${networkInterface}:off:8.8.8.8"
|
|
];
|
|
boot.initrd.availableKernelModules = [ networkInterfaceModule ];
|
|
boot.initrd.network.enable = true;
|
|
boot.initrd.network.ssh = {
|
|
enable = true;
|
|
authorizedKeys = config.users.users.root.openssh.authorizedKeys.keys;
|
|
port = 22;
|
|
hostKeys = [
|
|
/etc/secrets/initrd/ssh_host_rsa_key
|
|
/etc/secrets/initrd/ssh_host_ed25519_key
|
|
];
|
|
|
|
};
|
|
|
|
|
|
# make sure ip address is set after in initrd
|
|
#boot.initrd.network.postCommands = ''
|
|
# up ip addr add $IP_V4/32 dev eth0
|
|
# #ip address add $IP_V4/32 dev eth0
|
|
# #ip link set eth0 up
|
|
# #ip address add $IP_V4/32 dev $NIXOS_INTERFACE
|
|
# #ip link set eth0 up
|
|
#'';
|
|
|
|
|
|
}
|
|
EOF
|
|
|
|
|
|
mkdir -p /mnt/etc/secrets/initrd/
|
|
|
|
ssh-keygen -t rsa -N "" -f /mnt/etc/secrets/initrd/ssh_host_rsa_key
|
|
ssh-keygen -t ed25519 -N "" -f /mnt/etc/secrets/initrd/ssh_host_ed25519_key
|
|
|
|
|
|
# Install NixOS
|
|
#PATH="$PATH" NIX_PATH="$NIX_PATH" `which nixos-install` --no-root-passwd --root /mnt --max-jobs 40
|
|
PATH="$PATH" `which nixos-install` --no-root-passwd --root /mnt --max-jobs 40
|
|
|
|
umount /mnt/boot-{1,2}
|
|
umount /mnt
|
|
|
|
echo "Determined NIXOS_INTERFACE as '$NIXOS_INTERFACE'"
|
|
echo "Determined IP_V4 as $IP_V4"
|
|
echo "Determined IP_V6 as $IP_V6"
|
|
echo "Determined DEFAULT_GATEWAY as $DEFAULT_GATEWAY"
|
|
reboot
|