chore(deps): lock file maintenance

.github/workflows/ci.yml

@@ -25,11 +25,22 @@ jobs:
           attic use "$CACHE_REPOSITORY"
       - name: Run Builds and Checks
         run: nix-fast-build --no-nom --max-jobs 6 --skip-cached --attic-cache "$CACHE_REPOSITORY"
-      - name: Deploy Docs
-        if: success() && github.ref == 'refs/heads/main'
-        run: |
-          mkdir ~/.ssh/
-          echo -e "Host lindberg-webapps.backplane.net.qo.is\n    StrictHostKeyChecking no" >> ~/.ssh/config
-          (umask 0077 && printf "%s\n" "${{ secrets.SSH_DEPLOY_KEY }}" > ~/.ssh/id_ed25519)
-          deploy --skip-checks --remote-build .#lindberg-webapps.\"docs-ops.qo.is\"
-          # Remote build is neccessary due to non-wheel nix users signing restrictions. However, the build should come from the cache anyway.
+  deploy:
+    needs: build
+    if: success() && github.ref == 'refs/heads/main'
+    runs-on: nix
+    env:
+      SSH_DEPLOY_KEY: "${{ secrets.SSH_DEPLOY_KEY }}"
+    strategy:
+      matrix:
+        profile:
+          - docs-ops.qo.is
+          - system-vm
+    steps:
+      - name: Initialize CI
+        uses: https://git.qo.is/qo.is/actions-nix-init@main
+        with:
+          token: ${{ secrets.CI_TOKEN }}
+          lfs: false
+      - name: "Deploy profile"
+        run: "auto-deploy ${{ matrix.profile }}"

deploy/README.md

@@ -4,14 +4,26 @@ Note that you have to be connected to the `vpn.qo.is`
 (or execute the deployment from a host that is in the `backplane.net.qo.is` overlay network)
 and that you need to have SSH root access to the target machines.
 
+## Deploy system categories
+
+We currently split out nixosConfigurations into these categories:
+
+- `system-ci`: Systems should be updated separately because they might break automated deployment processes.
+- `system-vm`: Virtual systems.
+- `system-physical`: Physical systems.
+
+You can roll updates with retries by category with:
+
+```bash
+auto-deploy system-vm
+auto-deploy system-physical
+```
+
 ## Deploy to selected target hosts
 
 ```bash
-nix run .#deploy-qois .#<hostname>.system .#<hostname2>.system
-```
+nix develop
 
-## Deploy with extended timeouts (sometimes required for slow APU devices)
-
-```bash
-nix run .#deploy-qois  .#calanda.system -- --confirm-timeout 600 --activation-timeout 600
+deploy --skip-checks .#cyprianspitz.system-physical
+deploy --skip-checks .#lindberg-build.system-vm
 ```

deploy/docs-ops/default.nix

@@ -12,5 +12,6 @@ in
     sshUser = "nginx-${domain}";
     path = deployPkgs.deploy-rs.lib.activate.noop self.packages.${system}.docs;
     profilePath = "/var/lib/nginx-${domain}/root";
+    remoteBuild = true;
   };
 }

deploy/system-ci/default.nix

@@ -0,0 +1,27 @@
+{
+  deployPkgs,
+  pkgs,
+  self,
+  ...
+}:
+let
+  inherit (pkgs.lib) pipe filterAttrs mapAttrs;
+in
+{
+  nodes = pipe self.nixosConfigurations [
+    (filterAttrs (_n: v: v.config.qois.git-ci-runner.enable))
+    (mapAttrs (
+      host: config: {
+        hostname = "${host}.backplane.net.qo.is";
+        profiles.system-ci = {
+          sshUser = "root";
+          user = "root";
+          activationTimeout = 300;
+          confirmTimeout = 60;
+          remoteBuild = true;
+          path = deployPkgs.deploy-rs.lib.activate.nixos config;
+        };
+      }
+    ))
+  ];
+}

deploy/system-physical/default.nix

@@ -0,0 +1,27 @@
+{
+  deployPkgs,
+  pkgs,
+  self,
+  ...
+}:
+let
+  inherit (pkgs.lib) pipe filterAttrs mapAttrs;
+in
+{
+  nodes = pipe self.nixosConfigurations [
+    (filterAttrs (_n: v: !v.config.services.qemuGuest.enable && !v.config.qois.git-ci-runner.enable))
+    (mapAttrs (
+      host: config: {
+        hostname = "${host}.backplane.net.qo.is";
+        profiles.system-physical = {
+          sshUser = "root";
+          user = "root";
+          activationTimeout = 600;
+          confirmTimeout = 120;
+          remoteBuild = true;
+          path = deployPkgs.deploy-rs.lib.activate.nixos config;
+        };
+      }
+    ))
+  ];
+}

deploy/system-vm/default.nix

@@ -0,0 +1,27 @@
+{
+  deployPkgs,
+  pkgs,
+  self,
+  ...
+}:
+let
+  inherit (pkgs.lib) pipe filterAttrs mapAttrs;
+in
+{
+  nodes = pipe self.nixosConfigurations [
+    (filterAttrs (_n: v: v.config.services.qemuGuest.enable && !v.config.qois.git-ci-runner.enable))
+    (mapAttrs (
+      host: config: {
+        hostname = "${host}.backplane.net.qo.is";
+        profiles.system-vm = {
+          sshUser = "root";
+          user = "root";
+          activationTimeout = 300;
+          confirmTimeout = 60;
+          remoteBuild = true;
+          path = deployPkgs.deploy-rs.lib.activate.nixos config;
+        };
+      }
+    ))
+  ];
+}

deploy/system/default.nix

@@ -1,20 +0,0 @@
-{
-  deployPkgs,
-  pkgs,
-  self,
-  system,
-  ...
-}:
-{
-  nodes = pkgs.lib.mapAttrs (host: config: {
-    hostname = "${host}.backplane.net.qo.is";
-    profiles.system = {
-      sshUser = "root";
-      user = "root";
-      activationTimeout = 420;
-      confirmTimeout = 120;
-
-      path = deployPkgs.deploy-rs.lib.activate.nixos config;
-    };
-  }) self.nixosConfigurations;
-}

dev-shells/default.nix

@@ -29,9 +29,9 @@ in
       pre-commit-check.enabledPackages
       ++ [ vscodium-with-extensions ]
       ++ (with self.packages.${system}; [
-        deploy-qois
         sops
         sops-rekey
+        auto-deploy
       ])
       ++ (with pkgs; [
         attic-client

flake.lock

@@ -27,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1741786315,
-        "narHash": "sha256-VT65AE2syHVj6v/DGB496bqBnu1PXrrzwlw07/Zpllc=",
+        "lastModified": 1744940522,
+        "narHash": "sha256-TNoetfICvd29DhxRPpmyKItQBDlqSvKcV+wGNkn14jk=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "0d8c6ad4a43906d14abd5c60e0ffe7b587b213de",
+        "rev": "51d33bbb7f1e74ba5f9d9a77357735149da99081",
         "type": "github"
       },
       "original": {
@@ -133,11 +133,11 @@
     },
     "nixpkgs-nixos-stable": {
       "locked": {
-        "lastModified": 1743231893,
-        "narHash": "sha256-tpJsHMUPEhEnzySoQxx7+kA+KUtgWqvlcUBqROYNNt0=",
+        "lastModified": 1744440957,
+        "narHash": "sha256-FHlSkNqFmPxPJvy+6fNLaNeWnF1lZSgqVCl/eWaJRc4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "c570c1f5304493cafe133b8d843c7c1c4a10d3a6",
+        "rev": "26d499fc9f1d567283d5d56fcf367edd815dba1d",
         "type": "github"
       },
       "original": {
@@ -149,11 +149,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1743095683,
-        "narHash": "sha256-gWd4urRoLRe8GLVC/3rYRae1h+xfQzt09xOfb0PaHSk=",
+        "lastModified": 1744932701,
+        "narHash": "sha256-fusHbZCyv126cyArUwwKrLdCkgVAIaa/fQJYFlCEqiU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "5e5402ecbcb27af32284d4a62553c019a3a49ea6",
+        "rev": "b024ced1aac25639f8ca8fdfc2f8c4fbd66c48ef",
         "type": "github"
       },
       "original": {
@@ -170,10 +170,10 @@
         ]
       },
       "locked": {
-        "lastModified": 1742912717,
-        "narHash": "sha256-jKDVM4hLzNwYpg1/at53B2LZIEEvey1UUIi5cR8wNVQ=",
-        "rev": "80b04cd406adc46357888614ecc3f467b85ab8fa",
-        "revCount": 15,
+        "lastModified": 1743779182,
+        "narHash": "sha256-0wQ+22g6dfnIBIvw2Mji37R7OmCMrmqcp/4zSLQRKSQ=",
+        "rev": "8e7ec0d7f4a571a55d0e6b99fcd6f85fea474f1e",
+        "revCount": 16,
         "type": "git",
         "url": "file:./private"
       },
@@ -201,11 +201,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1743305778,
-        "narHash": "sha256-Ux/UohNtnM5mn9SFjaHp6IZe2aAnUCzklMluNtV6zFo=",
+        "lastModified": 1744669848,
+        "narHash": "sha256-pXyanHLUzLNd3MX9vsWG+6Z2hTU8niyphWstYEP3/GU=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "8e873886bbfc32163fe027b8676c75637b7da114",
+        "rev": "61154300d945f0b147b30d24ddcafa159148026a",
         "type": "github"
       },
       "original": {
@@ -236,11 +236,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1743081648,
-        "narHash": "sha256-WRAylyYptt6OX5eCEBWyTwOEqEtD6zt33rlUkr6u3cE=",
+        "lastModified": 1744961264,
+        "narHash": "sha256-aRmUh0AMwcbdjJHnytg1e5h5ECcaWtIFQa6d9gI85AI=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "29a3d7b768c70addce17af0869f6e2bd8f5be4b7",
+        "rev": "8d404a69efe76146368885110f29a2ca3700bee6",
         "type": "github"
       },
       "original": {

nixos-configurations/lindberg/README.md

@@ -2,9 +2,13 @@
 
 ## Operations {#\_operations}
 
-Reboot requires passphrase (see pass `host/lindberg/hdd_luks`)
+Reboot requires passphrase:
 
 ```bash
+# Get passphrase
+sops decrypt --extract '["system"]["hdd"]' private/nixos-configurations/lindberg/secrets.sops.yaml
+
+# Insert passphrase:
 ssh -p 2222 root@lindberg.riedbach-ext.net.qo.is
 ```
 

nixos-modules/system/virtual-machine.nix

@@ -13,6 +13,10 @@ with lib;
 
   config = lib.mkIf cfg.enable {
 
+    users.users.root.openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBS65v7n5ozOUjYGuO/dgLC9C5MUGL5kTnQnvWAYP5B3 ci@git.qo.is"
+    ]; # TODO: Move this key to allow CI deployment for all machines.
+
     boot.loader.grub.enable = true;
 
     system.autoUpgrade.allowReboot = true;

packages/auto-deploy/default.nix

@@ -0,0 +1,16 @@
+{
+  deploy-rs,
+  gitMinimal,
+  writeShellApplication,
+  lib,
+  ...
+}:
+writeShellApplication {
+  name = "auto-deploy";
+  meta.description = "Deploy machines automatically.";
+  runtimeInputs = [
+    deploy-rs
+    gitMinimal
+  ];
+  text = lib.readFile ./script.bash;
+}

packages/auto-deploy/script.bash

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+#### Environment
+FLAKE_ROOT="$(git rev-parse --show-toplevel)"
+
+export PROFILE="${1:-}"
+if [ -z "${PROFILE}" ]; then
+  echo "🛑 Error: No deployment profile was specified as first parameter (e.g. \"${0} system-vm\")" 1>&2
+  exit 1
+fi
+
+if [ -z "${SSH_DEPLOY_KEY:-}" ]; then
+  echo "ℹ️ Info: SSH_DEPLOY_KEY env variable was not set, ignoring."
+  SSH_KEY_FILE_ARG=""
+else
+  TEMP_KEY_FILE=$(mktemp /dev/shm/ssh_deploy_key.XXXXXXXX)
+  touch "${TEMP_KEY_FILE}" && chmod 600 "${TEMP_KEY_FILE}"
+  printf "%s\n" "${SSH_DEPLOY_KEY}" >"${TEMP_KEY_FILE}"
+  SSH_KEY_FILE_ARG="-i ${TEMP_KEY_FILE}"
+
+  # Set up a trap to remove the temporary key file on script exit
+  trap 'rm -f "${TEMP_KEY_FILE}"' EXIT
+  trap 'rm -f "${TEMP_KEY_FILE}"' SIGINT
+  trap 'rm -f "${TEMP_KEY_FILE}"' SIGTERM
+  trap 'rm -f "${TEMP_KEY_FILE}"' SIGQUIT
+fi
+
+HOSTS=$(nix eval --raw "${FLAKE_ROOT}"#deploy.nodes --apply "
+   nodes: let
+     inherit (builtins) attrNames filter concatStringsSep;
+     names = attrNames nodes;
+     profile = \"${PROFILE}\";
+     filteredNames = filter (name: nodes.\${name}.profiles ? \${profile}) names;
+   in concatStringsSep \"\\n\" filteredNames
+")
+if [ -z "$HOSTS" ]; then
+  echo "🛑 Error: No deployments matching the profile ${PROFILE} were found." 1>&2
+  exit 1
+fi
+
+KNOWN_HOSTS_FILE=$(nix eval --raw .#nixosConfigurations.lindberg.config.environment.etc."ssh/ssh_known_hosts".source)
+
+#### Helpers
+retry() {
+  local -r -i max_attempts="$1"
+  shift
+  local -i attempt_num=1
+  until "$@"; do
+    if ((attempt_num == max_attempts)); then
+      echo "🛑 Error: Attempt $attempt_num failed and there are no more attempts left!" 1>&2
+      return 1
+    else
+      echo "⚠️ Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+      sleep $((attempt_num++))
+    fi
+  done
+}
+
+#### Execution
+for HOST in $HOSTS; do
+  retry 3 deploy \
+    --skip-checks \
+    --ssh-opts "-o UserKnownHostsFile=${KNOWN_HOSTS_FILE} ${SSH_KEY_FILE_ARG:-}" \
+    --targets "${FLAKE_ROOT}#\"${HOST}\".\"${PROFILE}\""
+done

packages/deploy-qois/default.nix

@@ -1,14 +0,0 @@
-{
-  deploy-rs,
-  flakeSelf,
-  writeShellApplication,
-  ...
-}:
-writeShellApplication {
-  name = "deploy-qois";
-  meta.description = "Deploy configuration to specificed targets.";
-  runtimeInputs = [ deploy-rs ];
-  text = ''
-    deploy --interactive --targets "''${@:-${flakeSelf}}"
-  '';
-}

private

@@ -1 +1 @@
-Subproject commit 80b04cd406adc46357888614ecc3f467b85ab8fa
+Subproject commit 8e7ec0d7f4a571a55d0e6b99fcd6f85fea474f1e

renovate.json

@@ -3,11 +3,12 @@
   "extends": [
     "config:recommended"
   ],
+  "schedule": [
+    "* 18-19 * * *"
+  ],
   "lockFileMaintenance": {
     "enabled": true,
-    "extends": [
-      "schedule:weekly"
-    ]
+    "automerge": true
   },
   "cloneSubmodules": true,
   "nix": {

treefmt.nix

@@ -19,11 +19,13 @@
         "*.toml"
       ]
       ++ [
-        ".envrc"
-        "robots.txt"
         ".vscode/*"
         "nixos-modules/system/etc/*"
+        "private"
         "private/*"
+
+        ".envrc"
+        "robots.txt"
       ];
     formatter.jsonfmt.excludes = [ ".vscode/*.json" ];
   };

updates.md

@@ -22,13 +22,14 @@ Deploy updates:
 nix develop
 
 # Deploy vms
-deploy-qois .#lindberg-nextcloud .#lindberg-build
+auto-deploy system-vm
 
-# Deploy fast physical hosts
-deploy-qois .#lindberg
+# Deploy CI hosts
+auto-deploy system-ci
+
+# Deploy physical hosts
+auto-deploy system-physical
 
-# Deploy slow physical hosts (maybe do individually)
-deploy-qois --confirm-timeout 600 --activation-timeout 600 --targets .#stompert .#stompert
 
 ```