TMP: Deploy from deploy-vms-branch

.github/workflows/ci.yml

@@ -25,11 +25,17 @@ jobs:
           attic use "$CACHE_REPOSITORY"
       - name: Run Builds and Checks
         run: nix-fast-build --no-nom --max-jobs 6 --skip-cached --attic-cache "$CACHE_REPOSITORY"
-      - name: Deploy Docs
+  deploy:
-        if: success() && github.ref == 'refs/heads/main'
+    needs: build
-        run: |
+    if: success() && github.ref == 'refs/heads/54-deploy-vms-automatically'
-          mkdir ~/.ssh/
+    runs-on: nix
-          echo -e "Host lindberg-webapps.backplane.net.qo.is\n    StrictHostKeyChecking no" >> ~/.ssh/config
+    env:
-          (umask 0077 && printf "%s\n" "${{ secrets.SSH_DEPLOY_KEY }}" > ~/.ssh/id_ed25519)
+      SSH_DEPLOY_KEY: "${{ secrets.SSH_DEPLOY_KEY }}"
-          deploy --skip-checks --remote-build .#lindberg-webapps.\"docs-ops.qo.is\"
+    strategy:
-          # Remote build is neccessary due to non-wheel nix users signing restrictions. However, the build should come from the cache anyway.
+      matrix:
+        profile:
+          - docs-ops.qo.is
+          - system-vm
+    steps:
+      - name: "Deploy Profile ${{ matrix.profile }}"
+        run: "auto-deploy ${{ matrix.profile }}"

deploy/README.md

@@ -6,12 +6,17 @@ and that you need to have SSH root access to the target machines.
 
 ## Deploy system categories
 
-This is also used in CI.
+We currently split out nixosConfigurations into these categories:
+
+- `system-ci`: Systems should be updated separately because they might break automated deployment processes.
+- `system-vm`: Virtual systems.
+- `system-physical`: Physical systems.
+
+You can roll updates with retries by category with:
 
 ```bash
-
+auto-deploy system-vm
-auto-deploy vm
+auto-deploy system-physical
-auto-deploy physical
 ```
 
 ## Deploy to selected target hosts

deploy/docs-ops/default.nix

@@ -12,5 +12,6 @@ in
     sshUser = "nginx-${domain}";
     path = deployPkgs.deploy-rs.lib.activate.noop self.packages.${system}.docs;
     profilePath = "/var/lib/nginx-${domain}/root";
+    remoteBuild = true;
   };
 }

deploy/system-ci/default.nix

@@ -0,0 +1,27 @@
+{
+  deployPkgs,
+  pkgs,
+  self,
+  ...
+}:
+let
+  inherit (pkgs.lib) pipe filterAttrs mapAttrs;
+in
+{
+  nodes = pipe self.nixosConfigurations [
+    (filterAttrs (_n: v: v.config.qois.git-ci-runner.enable))
+    (mapAttrs (
+      host: config: {
+        hostname = "${host}.backplane.net.qo.is";
+        profiles.system-ci = {
+          sshUser = "root";
+          user = "root";
+          activationTimeout = 300;
+          confirmTimeout = 60;
+          remoteBuild = true;
+          path = deployPkgs.deploy-rs.lib.activate.nixos config;
+        };
+      }
+    ))
+  ];
+}

deploy/system-physical/default.nix

@@ -9,7 +9,7 @@ let
 in
 {
   nodes = pipe self.nixosConfigurations [
-    (filterAttrs (_n: v: v.config.services.qemuGuest.enable == false))
+    (filterAttrs (_n: v: !v.config.services.qemuGuest.enable && !v.config.qois.git-ci-runner.enable))
     (mapAttrs (
       host: config: {
         hostname = "${host}.backplane.net.qo.is";

deploy/system-vm/default.nix

@@ -9,7 +9,7 @@ let
 in
 {
   nodes = pipe self.nixosConfigurations [
-    (filterAttrs (_n: v: v.config.services.qemuGuest.enable))
+    (filterAttrs (_n: v: v.config.services.qemuGuest.enable && !v.config.qois.git-ci-runner.enable))
     (mapAttrs (
       host: config: {
         hostname = "${host}.backplane.net.qo.is";

packages/auto-deploy/script.bash

@@ -3,25 +3,42 @@
 #### Environment
 FLAKE_ROOT="$(git rev-parse --show-toplevel)"
 
-export PROFILE=""
+export PROFILE="${1:-}"
-case "${1:-''}" in
+if [ -z "${PROFILE}" ]; then
-vm | physical)
+  echo "🛑 Error: No deployment profile was specified as first parameter (e.g. \"${0} system-vm\")" 1>&2
-  PROFILE="system-$1"
-  ;;
-*)
-  echo "🛑 Error: Please use 'vm' or 'physical' as first parameter."
   exit 1
-  ;;
+fi
-esac
 
-HOSTS=$(nix eval --raw "$FLAKE_ROOT"#deploy.nodes --apply "
+if [ -z "${SSH_DEPLOY_KEY:-}" ]; then
+  echo "ℹ️ Info: SSH_DEPLOY_KEY env variable was not set, ignoring."
+  SSH_KEY_FILE_ARG=""
+else
+  TEMP_KEY_FILE=$(mktemp /dev/shm/ssh_deploy_key.XXXXXXXX)
+  touch "${TEMP_KEY_FILE}" && chmod 600 "${TEMP_KEY_FILE}"
+  printf "%s\n" "${SSH_DEPLOY_KEY}" >"${TEMP_KEY_FILE}"
+  SSH_KEY_FILE_ARG="-i ${TEMP_KEY_FILE}"
+
+  # Set up a trap to remove the temporary key file on script exit
+  trap 'rm -f "${TEMP_KEY_FILE}"' EXIT
+  trap 'rm -f "${TEMP_KEY_FILE}"' SIGINT
+  trap 'rm -f "${TEMP_KEY_FILE}"' SIGTERM
+  trap 'rm -f "${TEMP_KEY_FILE}"' SIGQUIT
+fi
+
+HOSTS=$(nix eval --raw "${FLAKE_ROOT}"#deploy.nodes --apply "
    nodes: let
      inherit (builtins) attrNames filter concatStringsSep;
      names = attrNames nodes;
-     profile = \"$PROFILE\";
+     profile = \"${PROFILE}\";
      filteredNames = filter (name: nodes.\${name}.profiles ? \${profile}) names;
    in concatStringsSep \"\\n\" filteredNames
 ")
+if [ -z "$HOSTS" ]; then
+  echo "🛑 Error: No deployments matching the profile ${PROFILE} were found." 1>&2
+  exit 1
+fi
+
+KNOWN_HOSTS_FILE=$(nix eval --raw .#nixosConfigurations.lindberg.config.environment.etc."ssh/ssh_known_hosts".source)
 
 #### Helpers
 retry() {
@@ -30,7 +47,7 @@ retry() {
   local -i attempt_num=1
   until "$@"; do
     if ((attempt_num == max_attempts)); then
-      echo "⚠️ Warning: Attempt $attempt_num failed and there are no more attempts left!"
+      echo "🛑 Error: Attempt $attempt_num failed and there are no more attempts left!" 1>&2
       return 1
     else
       echo "⚠️ Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
@@ -41,5 +58,8 @@ retry() {
 
 #### Execution
 for HOST in $HOSTS; do
-  retry 3 deploy --skip-checks --targets "${FLAKE_ROOT}#${HOST}.${PROFILE}"
+  retry 3 deploy \
+    --skip-checks \
+    --ssh-opts "-o UserKnownHostsFile=${KNOWN_HOSTS_FILE} ${SSH_KEY_FILE_ARG:-''}" \
+    --targets "${FLAKE_ROOT}#\"${HOST}\".\"${PROFILE}\""
 done

updates.md

@@ -22,10 +22,14 @@ Deploy updates:
 nix develop
 
 # Deploy vms
-auto-deploy vm
+auto-deploy system-vm
+
+# Deploy CI hosts
+auto-deploy system-ci
 
 # Deploy physical hosts
-auto-deploy physical
+auto-deploy system-physical
+
 
 ```