From 0bfbd5c2b83859f122bf5ef24df78f5369a68ba7 Mon Sep 17 00:00:00 2001 From: Ingolf Wagner Date: Fri, 4 Sep 2020 00:49:35 +0200 Subject: [PATCH] backup: restic -> borg --- assets/etc-info/restic.sh | 22 ---------- configs/mobi/configuration.nix | 2 - configs/mobi/restic.nix | 8 ---- configs/pepe/borg.nix | 15 +++++++ configs/pepe/configuration.nix | 2 + configs/pepe/dms.nix | 2 +- configs/porani/configuration.nix | 6 --- configs/porani/syncthing.nix | 2 +- configs/sputnik/configuration.nix | 2 - configs/sputnik/restic.nix | 8 ---- configs/workhorse/borg.nix | 6 +-- configs/workhorse/gogs.nix | 4 +- configs/workhorse/mail-fetcher.nix | 3 +- configs/workhorse/mattermost.nix | 2 +- configs/workhorse/syncthing.nix | 2 +- configs/workhorse/weechat.nix | 2 +- system/all/borg-jobs.nix | 54 ++++++++++++++++++++++++ system/all/default.nix | 3 +- system/all/on-failure.nix | 6 +-- system/all/restic.nix | 68 ------------------------------ system/all/sftp-user.nix | 17 -------- system/desktop/default.nix | 3 +- system/desktop/packages.nix | 9 +--- system/desktop/restic.nix | 7 --- 24 files changed, 90 insertions(+), 165 deletions(-) delete mode 100644 assets/etc-info/restic.sh delete mode 100644 configs/mobi/restic.nix create mode 100644 configs/pepe/borg.nix delete mode 100644 configs/sputnik/restic.nix create mode 100644 system/all/borg-jobs.nix delete mode 100644 system/all/restic.nix delete mode 100644 system/all/sftp-user.nix delete mode 100644 system/desktop/restic.nix diff --git a/assets/etc-info/restic.sh b/assets/etc-info/restic.sh deleted file mode 100644 index 76e60c2..0000000 --- a/assets/etc-info/restic.sh +++ /dev/null @@ -1,22 +0,0 @@ - -function only_service(){ - local service=$1 - cat ~/.etc_info/* | jq '.[] | select(has("'${service}'"))' -} - -only_service "restic" \ - | jq '.restic' | jq --slurp . | jq 'group_by(.from)' \ - | jq --raw-output ' -.[] | -.[0].from as $hostname | -.[0].folders as $folders | -[.[] | select(.enable) | .to.server ] as $servers | -" -# \($hostname) -## folders -\( $folders | reduce .[] as $item ( ""; . + "* " + $item + "\n" ) ) -## to -\( $servers | reduce .[] as $item ( ""; . + "* " + $item + "\n") ) -"' | pandoc-from-markdown-to-man | man --local-file - - - diff --git a/configs/mobi/configuration.nix b/configs/mobi/configuration.nix index ea38295..2e5a1df 100644 --- a/configs/mobi/configuration.nix +++ b/configs/mobi/configuration.nix @@ -4,8 +4,6 @@ ./hardware-configuration.nix - - ./restic.nix ./tinc.nix ]; diff --git a/configs/mobi/restic.nix b/configs/mobi/restic.nix deleted file mode 100644 index 2ef97d9..0000000 --- a/configs/mobi/restic.nix +++ /dev/null @@ -1,8 +0,0 @@ -{ - backup.services.restic = { - "on-pepe.insecure".enable = false; - "on-porani.insecure".enable = false; - "on-workhorse.private".enable = false; - "on-workout.private".enable = false; - }; -} diff --git a/configs/pepe/borg.nix b/configs/pepe/borg.nix new file mode 100644 index 0000000..c135954 --- /dev/null +++ b/configs/pepe/borg.nix @@ -0,0 +1,15 @@ +{ lib, config, pkgs, ... }: { + + # provide borg backup repository + services.borgbackup.repos = { + default = { + quota = "100G"; + allowSubRepos = true; + authorizedKeys = [ + # todo rename + (lib.fileContents ) + ]; + }; + }; + +} diff --git a/configs/pepe/configuration.nix b/configs/pepe/configuration.nix index 6127325..88d5a42 100644 --- a/configs/pepe/configuration.nix +++ b/configs/pepe/configuration.nix @@ -13,6 +13,8 @@ ./lan.nix ./dms.nix + ./borg.nix + ]; networking.hostName = "pepe"; diff --git a/configs/pepe/dms.nix b/configs/pepe/dms.nix index ac56c4f..a84e977 100644 --- a/configs/pepe/dms.nix +++ b/configs/pepe/dms.nix @@ -90,6 +90,6 @@ in { }; # add documents to backup - backup.all.restic.dirs = [ "/home/ftp-upload/db" ]; + backup.dirs = [ "/home/ftp-upload/db" ]; } diff --git a/configs/porani/configuration.nix b/configs/porani/configuration.nix index f867607..74b7505 100644 --- a/configs/porani/configuration.nix +++ b/configs/porani/configuration.nix @@ -12,12 +12,6 @@ networking.hostName = "porani"; - backup.services.restic = { - "on-porani.insecure".enable = false; - "on-workhorse.private".enable = false; - "on-workout.private".enable = false; - }; - # not needed because not encrypted # enable initrd ssh #configuration.init-ssh = { diff --git a/configs/porani/syncthing.nix b/configs/porani/syncthing.nix index 040f8df..c5548d1 100644 --- a/configs/porani/syncthing.nix +++ b/configs/porani/syncthing.nix @@ -58,7 +58,7 @@ users.groups."syncthing".members = [ "mpd" "syncthing" "kodi" "palo" ]; - backup.all.restic.dirs = [ "/var/lib/syncthing/finance" ]; + backup.dirs = [ "/var/lib/syncthing/finance" ]; } diff --git a/configs/sputnik/configuration.nix b/configs/sputnik/configuration.nix index c7f78ac..d2f20b0 100644 --- a/configs/sputnik/configuration.nix +++ b/configs/sputnik/configuration.nix @@ -6,9 +6,7 @@ ./nginx.nix ./tinc.nix - ./restic.nix #./syncplay.nix - #./mail-server.nix ]; diff --git a/configs/sputnik/restic.nix b/configs/sputnik/restic.nix deleted file mode 100644 index 18c216a..0000000 --- a/configs/sputnik/restic.nix +++ /dev/null @@ -1,8 +0,0 @@ -{ - backup.services.restic = { - "on-pepe.private".enable = false; - "on-porani.insecure".enable = false; - "on-workhorse.private".enable = false; - "on-workout.private".enable = false; - }; -} diff --git a/configs/workhorse/borg.nix b/configs/workhorse/borg.nix index b1a2c6e..9436a0f 100644 --- a/configs/workhorse/borg.nix +++ b/configs/workhorse/borg.nix @@ -4,9 +4,9 @@ default = { quota = "100G"; allowSubRepos = true; - authorizedKeysAppendOnly = [ - # todo rename this one - (toString ) + authorizedKeys = [ + # todo rename + (lib.fileContents ) ]; }; }; diff --git a/configs/workhorse/gogs.nix b/configs/workhorse/gogs.nix index 7d8cd59..9eca11a 100644 --- a/configs/workhorse/gogs.nix +++ b/configs/workhorse/gogs.nix @@ -73,7 +73,7 @@ in { LEVEL = Warn ''; }; - backup.all.restic.dirs = [ config.services.gogs.repositoryRoot ]; + backup.dirs = [ config.services.gogs.repositoryRoot ]; #services.gitea = { # enable = true; @@ -99,6 +99,6 @@ in { # LEVEL = Warn # ''; #}; - #backup.all.restic.dirs = [ config.services.gitea.repositoryRoot ]; + #backup.dirs = [ config.services.gitea.repositoryRoot ]; } diff --git a/configs/workhorse/mail-fetcher.nix b/configs/workhorse/mail-fetcher.nix index 98bbb88..2d875fe 100644 --- a/configs/workhorse/mail-fetcher.nix +++ b/configs/workhorse/mail-fetcher.nix @@ -455,13 +455,14 @@ let in { - backup.all.restic.dirs = [ config.users.users.mailUser.home ]; + backup.dirs = [ "/home/mailfetcher" ]; users.users.mailUser = { isNormalUser = true; description = "collects mails for me"; hashedPassword = "!"; name = "mailfetcher"; + home = "/home/mailfetcher"; openssh.authorizedKeys.keyFiles = config.users.users.root.openssh.authorizedKeys.keyFiles; }; diff --git a/configs/workhorse/mattermost.nix b/configs/workhorse/mattermost.nix index 0622265..1b05e28 100644 --- a/configs/workhorse/mattermost.nix +++ b/configs/workhorse/mattermost.nix @@ -7,7 +7,7 @@ let in { # backup mattermost - backup.all.restic.dirs = [ "/home/mattermost" ]; + backup.dirs = [ "/home/mattermost" ]; containers.mattermost = { diff --git a/configs/workhorse/syncthing.nix b/configs/workhorse/syncthing.nix index 35b9fad..3f9ed05 100644 --- a/configs/workhorse/syncthing.nix +++ b/configs/workhorse/syncthing.nix @@ -116,6 +116,6 @@ after = [ "media.mount" ]; }; - backup.all.restic.dirs = [ "/home/syncthing/finance" ]; + backup.dirs = [ "/home/syncthing/finance" ]; } diff --git a/configs/workhorse/weechat.nix b/configs/workhorse/weechat.nix index 50e35bc..f008033 100644 --- a/configs/workhorse/weechat.nix +++ b/configs/workhorse/weechat.nix @@ -33,6 +33,6 @@ # otherwise xterm is the only thing that works environment.systemPackages = [ pkgs.rxvt_unicode ]; - backup.all.restic.dirs = [ config.services.weechat.root ]; + backup.dirs = [ config.services.weechat.root ]; } diff --git a/system/all/borg-jobs.nix b/system/all/borg-jobs.nix new file mode 100644 index 0000000..5228655 --- /dev/null +++ b/system/all/borg-jobs.nix @@ -0,0 +1,54 @@ +{ config, lib, ... }: { + + options = { + backup.dirs = lib.mkOption { + default = [ ]; + type = with lib.types; listOf str; + }; + }; + + config = let + servers = [ + { + name = "workhorse"; + host = "workhorse.private"; + } + { + name = "pepe"; + host = "pepe.private"; + } + ]; + + dirs = config.backup.dirs; + + myHostname = config.networking.hostName; + + setup = server: { + paths = dirs; + doInit = true; + repo = "borg@${server}:./${myHostname}"; + encryption = { + mode = "repokey-blake2"; + # todo rename + passCommand = "cat ${toString }"; + }; + environment.BORG_RSH = + "ssh -i ${toString }"; + compression = "auto,lzma"; + startAt = "daily"; + + }; + + in { + + services.borgbackup.jobs = let + setups = map ({ name, host }: { "${name}" = setup host; }) servers; + setupAttrs = lib.zipAttrsWith (_: vals: lib.head vals) setups; + nonEmptySetups = + lib.filterAttrs (_: { paths, ... }: builtins.length paths != 0) + setupAttrs; + in nonEmptySetups; + + }; + +} diff --git a/system/all/default.nix b/system/all/default.nix index ff0dbb3..8a5caf1 100644 --- a/system/all/default.nix +++ b/system/all/default.nix @@ -17,8 +17,7 @@ ./nginx-landingpage.nix ./nginx.nix ./packages.nix - ./restic.nix - ./sftp-user.nix + ./borg-jobs.nix ./sshd-known-hosts-bootup.nix ./sshd-known-hosts-private.nix ./sshd-known-hosts-public.nix diff --git a/system/all/on-failure.nix b/system/all/on-failure.nix index ca1f626..bdc0082 100644 --- a/system/all/on-failure.nix +++ b/system/all/on-failure.nix @@ -8,9 +8,9 @@ sshd.name = "sshd"; tor.name = "tor"; dnsmasq.name = "dnsmasq"; - backup_on_workhorse.name = "backup.on-workhorse.private"; - backup_on_workout.name = "backup.on-workout.private"; - backup_on_porani.name = "backup.on-porani.private"; + #backup_on_workhorse.name = "backup.on-workhorse.private"; + #backup_on_workout.name = "backup.on-workout.private"; + #backup_on_porani.name = "backup.on-porani.private"; syncthing.name = "syncthing"; }; }; diff --git a/system/all/restic.nix b/system/all/restic.nix deleted file mode 100644 index fcc5bed..0000000 --- a/system/all/restic.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ config, lib, ... }: { - - options = { - backup.all.restic.dirs = lib.mkOption { - default = [ ]; - type = with lib.types; listOf str; - }; - }; - - config = let - servers = [ - "porani.insecure" - "workhorse.private" - "workout.private" - "pepe.private" - ]; - dirs = config.backup.all.restic.dirs; - - setup = server: { - enable = lib.mkDefault true; - passwordFile = toString ; - repo = "sftp::remote/remote-${config.networking.hostName}"; - requires = [ ]; - extraArguments = [ - "sftp.command='ssh backup@${server} -i ${ - toString - } -s sftp'" - ]; - initialize = true; - timerConfig = { - OnCalendar = "daily"; - Persistent = "true"; - }; - dirs = dirs; - }; - - hostname = config.networking.hostName; - infoEntry = server: { - restic = { - folders = dirs; - from = hostname; - to = { - server = server; - repo = config.backup.services.restic."on-${server}".repo; - }; - enable = config.backup.services.restic."on-${server}".enable; - }; - }; - - in { - - backup.services.restic = lib.zipAttrsWith (name: vals: lib.head vals) - (map (server: { "on-${server}" = setup server; }) servers); - - systemd.services = let - timeoutConfig = server: { - name = "backup.on-${server}"; - value = { serviceConfig.TimeoutSec = 30 * 60; }; - }; - in builtins.listToAttrs (map timeoutConfig servers); - - environment.etc."info/restic-${hostname}.json" = { - enable = true; - text = builtins.toJSON (map infoEntry servers); - }; - }; - -} diff --git a/system/all/sftp-user.nix b/system/all/sftp-user.nix deleted file mode 100644 index f6efeec..0000000 --- a/system/all/sftp-user.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ config, pkgs, lib, ... }: { - - module.backup.sftpUser = { - - # my backup user - "backup" = { - enable = true; - initialize = true; - home = toString /backup/remote; - authorizedKeys.keyFiles = - [ (toString ) ] - ++ config.users.users.root.openssh.authorizedKeys.keyFiles; - }; - - }; - -} diff --git a/system/desktop/default.nix b/system/desktop/default.nix index 8e89d4d..1d89b6e 100644 --- a/system/desktop/default.nix +++ b/system/desktop/default.nix @@ -16,7 +16,6 @@ ./packages.nix ./pass.nix ./remote-install.nix - ./restic.nix ./size.nix ./sshd.nix ./suspend.nix @@ -27,6 +26,8 @@ ./wtf.nix ]; + backup.dirs = [ "/home/palo/.password-store" ]; + programs.custom = { urxvt = { diff --git a/system/desktop/packages.nix b/system/desktop/packages.nix index e77a363..3171764 100644 --- a/system/desktop/packages.nix +++ b/system/desktop/packages.nix @@ -194,7 +194,7 @@ in { aspellDicts.es translate-shell - restic + borgbackup gpa gnupg @@ -322,13 +322,6 @@ in { (map (host: "rsync -avLz ${host}.private:/etc/info/ ~/.etc_info") (attrNames config.module.cluster.services.tinc."private".hosts)))) - (pkgs.writers.writeBashBin "etc-info-restic" - (lib.fileContents )) - - #(pkgs.writers.writeDashBin "reddit" '' - #${unstablePkgs.tuir}/bin/tuir "$@" - #'') - ] ++ (lib.crossLists pandocScript [ [ "man" "markdown" "mediawiki" ] [ "mediawiki" "docbook5" "html5" "man" ] diff --git a/system/desktop/restic.nix b/system/desktop/restic.nix deleted file mode 100644 index 1a29092..0000000 --- a/system/desktop/restic.nix +++ /dev/null @@ -1,7 +0,0 @@ -{ lib, ... }: { - - backup.services.restic = { "on-porani.insecure".enable = false; }; - - backup.all.restic.dirs = [ "/home/palo/.password-store" ]; - -}