diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d886cc..c40fdf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,22 +25,11 @@ jobs: attic use "$CACHE_REPOSITORY" - name: Run Builds and Checks run: nix-fast-build --no-nom --max-jobs 6 --skip-cached --attic-cache "$CACHE_REPOSITORY" - deploy: - needs: build - if: success() && github.ref == 'refs/heads/main' - runs-on: nix - env: - SSH_DEPLOY_KEY: "${{ secrets.SSH_DEPLOY_KEY }}" - strategy: - matrix: - profile: - - docs-ops.qo.is - - system-vm - steps: - - name: Initialize CI - uses: https://git.qo.is/qo.is/actions-nix-init@main - with: - token: ${{ secrets.CI_TOKEN }} - lfs: false - - name: "Deploy profile" - run: "auto-deploy ${{ matrix.profile }}" + - name: Deploy Docs + if: success() && github.ref == 'refs/heads/main' + run: | + mkdir ~/.ssh/ + echo -e "Host lindberg-webapps.backplane.net.qo.is\n StrictHostKeyChecking no" >> ~/.ssh/config + (umask 0077 && printf "%s\n" "${{ secrets.SSH_DEPLOY_KEY }}" > ~/.ssh/id_ed25519) + deploy --skip-checks --remote-build .#lindberg-webapps.\"docs-ops.qo.is\" + # Remote build is neccessary due to non-wheel nix users signing restrictions. However, the build should come from the cache anyway. diff --git a/deploy/README.md b/deploy/README.md index d0abdbc..8c95d8a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -4,26 +4,14 @@ Note that you have to be connected to the `vpn.qo.is` (or execute the deployment from a host that is in the `backplane.net.qo.is` overlay network) and that you need to have SSH root access to the target machines. -## Deploy system categories - -We currently split out nixosConfigurations into these categories: - -- `system-ci`: Systems should be updated separately because they might break automated deployment processes. -- `system-vm`: Virtual systems. -- `system-physical`: Physical systems. - -You can roll updates with retries by category with: - -```bash -auto-deploy system-vm -auto-deploy system-physical -``` - ## Deploy to selected target hosts ```bash -nix develop - -deploy --skip-checks .#cyprianspitz.system-physical -deploy --skip-checks .#lindberg-build.system-vm +nix run .#deploy-qois .#.system .#.system +``` + +## Deploy with extended timeouts (sometimes required for slow APU devices) + +```bash +nix run .#deploy-qois .#calanda.system -- --confirm-timeout 600 --activation-timeout 600 ``` diff --git a/deploy/docs-ops/default.nix b/deploy/docs-ops/default.nix index 8fd2890..7fcf9ae 100644 --- a/deploy/docs-ops/default.nix +++ b/deploy/docs-ops/default.nix @@ -12,6 +12,5 @@ in sshUser = "nginx-${domain}"; path = deployPkgs.deploy-rs.lib.activate.noop self.packages.${system}.docs; profilePath = "/var/lib/nginx-${domain}/root"; - remoteBuild = true; }; } diff --git a/deploy/system-ci/default.nix b/deploy/system-ci/default.nix deleted file mode 100644 index a9e49a0..0000000 --- a/deploy/system-ci/default.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ - deployPkgs, - pkgs, - self, - ... -}: -let - inherit (pkgs.lib) pipe filterAttrs mapAttrs; -in -{ - nodes = pipe self.nixosConfigurations [ - (filterAttrs (_n: v: v.config.qois.git-ci-runner.enable)) - (mapAttrs ( - host: config: { - hostname = "${host}.backplane.net.qo.is"; - profiles.system-ci = { - sshUser = "root"; - user = "root"; - activationTimeout = 300; - confirmTimeout = 60; - remoteBuild = true; - path = deployPkgs.deploy-rs.lib.activate.nixos config; - }; - } - )) - ]; -} diff --git a/deploy/system-physical/default.nix b/deploy/system-physical/default.nix deleted file mode 100644 index 4e60068..0000000 --- a/deploy/system-physical/default.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ - deployPkgs, - pkgs, - self, - ... -}: -let - inherit (pkgs.lib) pipe filterAttrs mapAttrs; -in -{ - nodes = pipe self.nixosConfigurations [ - (filterAttrs (_n: v: !v.config.services.qemuGuest.enable && !v.config.qois.git-ci-runner.enable)) - (mapAttrs ( - host: config: { - hostname = "${host}.backplane.net.qo.is"; - profiles.system-physical = { - sshUser = "root"; - user = "root"; - activationTimeout = 600; - confirmTimeout = 120; - remoteBuild = true; - path = deployPkgs.deploy-rs.lib.activate.nixos config; - }; - } - )) - ]; -} diff --git a/deploy/system-vm/default.nix b/deploy/system-vm/default.nix deleted file mode 100644 index 65177b4..0000000 --- a/deploy/system-vm/default.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ - deployPkgs, - pkgs, - self, - ... -}: -let - inherit (pkgs.lib) pipe filterAttrs mapAttrs; -in -{ - nodes = pipe self.nixosConfigurations [ - (filterAttrs (_n: v: v.config.services.qemuGuest.enable && !v.config.qois.git-ci-runner.enable)) - (mapAttrs ( - host: config: { - hostname = "${host}.backplane.net.qo.is"; - profiles.system-vm = { - sshUser = "root"; - user = "root"; - activationTimeout = 300; - confirmTimeout = 60; - remoteBuild = true; - path = deployPkgs.deploy-rs.lib.activate.nixos config; - }; - } - )) - ]; -} diff --git a/deploy/system/default.nix b/deploy/system/default.nix new file mode 100644 index 0000000..cdaf846 --- /dev/null +++ b/deploy/system/default.nix @@ -0,0 +1,20 @@ +{ + deployPkgs, + pkgs, + self, + system, + ... +}: +{ + nodes = pkgs.lib.mapAttrs (host: config: { + hostname = "${host}.backplane.net.qo.is"; + profiles.system = { + sshUser = "root"; + user = "root"; + activationTimeout = 420; + confirmTimeout = 120; + + path = deployPkgs.deploy-rs.lib.activate.nixos config; + }; + }) self.nixosConfigurations; +} diff --git a/dev-shells/default.nix b/dev-shells/default.nix index 6f289ff..2023f3e 100644 --- a/dev-shells/default.nix +++ b/dev-shells/default.nix @@ -29,9 +29,9 @@ in pre-commit-check.enabledPackages ++ [ vscodium-with-extensions ] ++ (with self.packages.${system}; [ + deploy-qois sops sops-rekey - auto-deploy ]) ++ (with pkgs; [ attic-client diff --git a/flake.lock b/flake.lock index a23e572..8f42d57 100644 --- a/flake.lock +++ b/flake.lock @@ -27,11 +27,11 @@ ] }, "locked": { - "lastModified": 1744940522, - "narHash": "sha256-TNoetfICvd29DhxRPpmyKItQBDlqSvKcV+wGNkn14jk=", + "lastModified": 1743598667, + "narHash": "sha256-ViE7NoFWytYO2uJONTAX35eGsvTYXNHjWALeHAg8OQY=", "owner": "nix-community", "repo": "disko", - "rev": "51d33bbb7f1e74ba5f9d9a77357735149da99081", + "rev": "329d3d7e8bc63dd30c39e14e6076db590a6eabe6", "type": "github" }, "original": { @@ -133,11 +133,11 @@ }, "nixpkgs-nixos-stable": { "locked": { - "lastModified": 1744440957, - "narHash": "sha256-FHlSkNqFmPxPJvy+6fNLaNeWnF1lZSgqVCl/eWaJRc4=", + "lastModified": 1743813633, + "narHash": "sha256-BgkBz4NpV6Kg8XF7cmHDHRVGZYnKbvG0Y4p+jElwxaM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "26d499fc9f1d567283d5d56fcf367edd815dba1d", + "rev": "7819a0d29d1dd2bc331bec4b327f0776359b1fa6", "type": "github" }, "original": { @@ -149,11 +149,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1744932701, - "narHash": "sha256-fusHbZCyv126cyArUwwKrLdCkgVAIaa/fQJYFlCEqiU=", + "lastModified": 1743827369, + "narHash": "sha256-rpqepOZ8Eo1zg+KJeWoq1HAOgoMCDloqv5r2EAa9TSA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b024ced1aac25639f8ca8fdfc2f8c4fbd66c48ef", + "rev": "42a1c966be226125b48c384171c44c651c236c22", "type": "github" }, "original": { @@ -170,10 +170,10 @@ ] }, "locked": { - "lastModified": 1743779182, - "narHash": "sha256-0wQ+22g6dfnIBIvw2Mji37R7OmCMrmqcp/4zSLQRKSQ=", - "rev": "8e7ec0d7f4a571a55d0e6b99fcd6f85fea474f1e", - "revCount": 16, + "lastModified": 1742912717, + "narHash": "sha256-jKDVM4hLzNwYpg1/at53B2LZIEEvey1UUIi5cR8wNVQ=", + "rev": "80b04cd406adc46357888614ecc3f467b85ab8fa", + "revCount": 15, "type": "git", "url": "file:./private" }, @@ -201,11 +201,11 @@ ] }, "locked": { - "lastModified": 1744669848, - "narHash": "sha256-pXyanHLUzLNd3MX9vsWG+6Z2hTU8niyphWstYEP3/GU=", + "lastModified": 1743910657, + "narHash": "sha256-zr2jmWeWyhCD8WmO2aWov2g0WPPuZfcJDKzMJZYGq3Y=", "owner": "Mic92", "repo": "sops-nix", - "rev": "61154300d945f0b147b30d24ddcafa159148026a", + "rev": "523f58a4faff6c67f5f685bed33a7721e984c304", "type": "github" }, "original": { @@ -236,11 +236,11 @@ ] }, "locked": { - "lastModified": 1744961264, - "narHash": "sha256-aRmUh0AMwcbdjJHnytg1e5h5ECcaWtIFQa6d9gI85AI=", + "lastModified": 1743748085, + "narHash": "sha256-uhjnlaVTWo5iD3LXics1rp9gaKgDRQj6660+gbUU3cE=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "8d404a69efe76146368885110f29a2ca3700bee6", + "rev": "815e4121d6a5d504c0f96e5be2dd7f871e4fd99d", "type": "github" }, "original": { diff --git a/nixos-configurations/lindberg/README.md b/nixos-configurations/lindberg/README.md index 62f8332..7ac97ea 100644 --- a/nixos-configurations/lindberg/README.md +++ b/nixos-configurations/lindberg/README.md @@ -2,13 +2,9 @@ ## Operations {#\_operations} -Reboot requires passphrase: +Reboot requires passphrase (see pass `host/lindberg/hdd_luks`) ```bash -# Get passphrase -sops decrypt --extract '["system"]["hdd"]' private/nixos-configurations/lindberg/secrets.sops.yaml - -# Insert passphrase: ssh -p 2222 root@lindberg.riedbach-ext.net.qo.is ``` diff --git a/nixos-modules/system/virtual-machine.nix b/nixos-modules/system/virtual-machine.nix index 776571e..5e5a8ae 100644 --- a/nixos-modules/system/virtual-machine.nix +++ b/nixos-modules/system/virtual-machine.nix @@ -13,10 +13,6 @@ with lib; config = lib.mkIf cfg.enable { - users.users.root.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBS65v7n5ozOUjYGuO/dgLC9C5MUGL5kTnQnvWAYP5B3 ci@git.qo.is" - ]; # TODO: Move this key to allow CI deployment for all machines. - boot.loader.grub.enable = true; system.autoUpgrade.allowReboot = true; diff --git a/packages/auto-deploy/default.nix b/packages/auto-deploy/default.nix deleted file mode 100644 index 3c266a7..0000000 --- a/packages/auto-deploy/default.nix +++ /dev/null @@ -1,16 +0,0 @@ -{ - deploy-rs, - gitMinimal, - writeShellApplication, - lib, - ... -}: -writeShellApplication { - name = "auto-deploy"; - meta.description = "Deploy machines automatically."; - runtimeInputs = [ - deploy-rs - gitMinimal - ]; - text = lib.readFile ./script.bash; -} diff --git a/packages/auto-deploy/script.bash b/packages/auto-deploy/script.bash deleted file mode 100644 index 442eb4c..0000000 --- a/packages/auto-deploy/script.bash +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -#### Environment -FLAKE_ROOT="$(git rev-parse --show-toplevel)" - -export PROFILE="${1:-}" -if [ -z "${PROFILE}" ]; then - echo "🛑 Error: No deployment profile was specified as first parameter (e.g. \"${0} system-vm\")" 1>&2 - exit 1 -fi - -if [ -z "${SSH_DEPLOY_KEY:-}" ]; then - echo "ℹī¸ Info: SSH_DEPLOY_KEY env variable was not set, ignoring." - SSH_KEY_FILE_ARG="" -else - TEMP_KEY_FILE=$(mktemp /dev/shm/ssh_deploy_key.XXXXXXXX) - touch "${TEMP_KEY_FILE}" && chmod 600 "${TEMP_KEY_FILE}" - printf "%s\n" "${SSH_DEPLOY_KEY}" >"${TEMP_KEY_FILE}" - SSH_KEY_FILE_ARG="-i ${TEMP_KEY_FILE}" - - # Set up a trap to remove the temporary key file on script exit - trap 'rm -f "${TEMP_KEY_FILE}"' EXIT - trap 'rm -f "${TEMP_KEY_FILE}"' SIGINT - trap 'rm -f "${TEMP_KEY_FILE}"' SIGTERM - trap 'rm -f "${TEMP_KEY_FILE}"' SIGQUIT -fi - -HOSTS=$(nix eval --raw "${FLAKE_ROOT}"#deploy.nodes --apply " - nodes: let - inherit (builtins) attrNames filter concatStringsSep; - names = attrNames nodes; - profile = \"${PROFILE}\"; - filteredNames = filter (name: nodes.\${name}.profiles ? \${profile}) names; - in concatStringsSep \"\\n\" filteredNames -") -if [ -z "$HOSTS" ]; then - echo "🛑 Error: No deployments matching the profile ${PROFILE} were found." 1>&2 - exit 1 -fi - -KNOWN_HOSTS_FILE=$(nix eval --raw .#nixosConfigurations.lindberg.config.environment.etc."ssh/ssh_known_hosts".source) - -#### Helpers -retry() { - local -r -i max_attempts="$1" - shift - local -i attempt_num=1 - until "$@"; do - if ((attempt_num == max_attempts)); then - echo "🛑 Error: Attempt $attempt_num failed and there are no more attempts left!" 1>&2 - return 1 - else - echo "⚠ī¸ Attempt $attempt_num failed! Trying again in $attempt_num seconds..." - sleep $((attempt_num++)) - fi - done -} - -#### Execution -for HOST in $HOSTS; do - retry 3 deploy \ - --skip-checks \ - --ssh-opts "-o UserKnownHostsFile=${KNOWN_HOSTS_FILE} ${SSH_KEY_FILE_ARG:-}" \ - --targets "${FLAKE_ROOT}#\"${HOST}\".\"${PROFILE}\"" -done diff --git a/packages/deploy-qois/default.nix b/packages/deploy-qois/default.nix new file mode 100644 index 0000000..ed647ef --- /dev/null +++ b/packages/deploy-qois/default.nix @@ -0,0 +1,14 @@ +{ + deploy-rs, + flakeSelf, + writeShellApplication, + ... +}: +writeShellApplication { + name = "deploy-qois"; + meta.description = "Deploy configuration to specificed targets."; + runtimeInputs = [ deploy-rs ]; + text = '' + deploy --interactive --targets "''${@:-${flakeSelf}}" + ''; +} diff --git a/private b/private index 8e7ec0d..80b04cd 160000 --- a/private +++ b/private @@ -1 +1 @@ -Subproject commit 8e7ec0d7f4a571a55d0e6b99fcd6f85fea474f1e +Subproject commit 80b04cd406adc46357888614ecc3f467b85ab8fa diff --git a/renovate.json b/renovate.json index 63404e8..f710e2d 100644 --- a/renovate.json +++ b/renovate.json @@ -3,12 +3,11 @@ "extends": [ "config:recommended" ], - "schedule": [ - "* 18-19 * * *" - ], "lockFileMaintenance": { "enabled": true, - "automerge": true + "extends": [ + "schedule:weekly" + ] }, "cloneSubmodules": true, "nix": { diff --git a/treefmt.nix b/treefmt.nix index 14412c6..5e44926 100644 --- a/treefmt.nix +++ b/treefmt.nix @@ -19,13 +19,11 @@ "*.toml" ] ++ [ - ".vscode/*" - "nixos-modules/system/etc/*" - "private" - "private/*" - ".envrc" "robots.txt" + ".vscode/*" + "nixos-modules/system/etc/*" + "private/*" ]; formatter.jsonfmt.excludes = [ ".vscode/*.json" ]; }; diff --git a/updates.md b/updates.md index 4a8f56d..949cac7 100644 --- a/updates.md +++ b/updates.md @@ -22,14 +22,13 @@ Deploy updates: nix develop # Deploy vms -auto-deploy system-vm +deploy-qois .#lindberg-nextcloud .#lindberg-build -# Deploy CI hosts -auto-deploy system-ci - -# Deploy physical hosts -auto-deploy system-physical +# Deploy fast physical hosts +deploy-qois .#lindberg +# Deploy slow physical hosts (maybe do individually) +deploy-qois --confirm-timeout 600 --activation-timeout 600 --targets .#stompert .#stompert ```