From 86a3ffc42d87831f38507e73419c933be3328ff0 Mon Sep 17 00:00:00 2001 From: Fabian Hauser Date: Mon, 31 Mar 2025 15:02:18 +0300 Subject: [PATCH 1/8] Add flags to deploy-qois --- packages/deploy-qois/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/deploy-qois/default.nix b/packages/deploy-qois/default.nix index ed647ef..49ab5e7 100644 --- a/packages/deploy-qois/default.nix +++ b/packages/deploy-qois/default.nix @@ -9,6 +9,6 @@ writeShellApplication { meta.description = "Deploy configuration to specificed targets."; runtimeInputs = [ deploy-rs ]; text = '' - deploy --interactive --targets "''${@:-${flakeSelf}}" + deploy --remote-build --skip-checks --interactive --targets "''${@:-${flakeSelf}}" ''; } From e3cacda3567f207ba2ad2fe237c3fa829f632989 Mon Sep 17 00:00:00 2001 From: Fabian Hauser Date: Fri, 4 Apr 2025 18:07:10 +0300 Subject: [PATCH 2/8] Update hdd encryption docs for lindberg --- flake.lock | 9 +++++---- nixos-configurations/lindberg/README.md | 6 +++++- private | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 0d50309..9af33a5 100644 --- a/flake.lock +++ b/flake.lock @@ -170,10 +170,11 @@ ] }, "locked": { - "lastModified": 1742912717, - "narHash": "sha256-jKDVM4hLzNwYpg1/at53B2LZIEEvey1UUIi5cR8wNVQ=", - "rev": "80b04cd406adc46357888614ecc3f467b85ab8fa", - "revCount": 15, + "lastModified": 1743779182, + "narHash": "sha256-0wQ+22g6dfnIBIvw2Mji37R7OmCMrmqcp/4zSLQRKSQ=", + "ref": "refs/heads/main", + "rev": "8e7ec0d7f4a571a55d0e6b99fcd6f85fea474f1e", + "revCount": 16, "type": "git", "url": "file:./private" }, diff --git a/nixos-configurations/lindberg/README.md b/nixos-configurations/lindberg/README.md index 7ac97ea..62f8332 100644 --- a/nixos-configurations/lindberg/README.md +++ b/nixos-configurations/lindberg/README.md @@ -2,9 +2,13 @@ ## Operations {#\_operations} -Reboot requires passphrase (see pass `host/lindberg/hdd_luks`) +Reboot requires passphrase: ```bash +# Get passphrase +sops decrypt --extract '["system"]["hdd"]' private/nixos-configurations/lindberg/secrets.sops.yaml + +# Insert passphrase: ssh -p 2222 root@lindberg.riedbach-ext.net.qo.is ``` diff --git a/private b/private index 80b04cd..8e7ec0d 160000 --- a/private +++ b/private @@ -1 +1 @@ -Subproject commit 80b04cd406adc46357888614ecc3f467b85ab8fa +Subproject commit 8e7ec0d7f4a571a55d0e6b99fcd6f85fea474f1e From 6a745892a4d7fd85704fbbf34b19920f3720b7c2 Mon Sep 17 00:00:00 2001 From: Fabian Hauser Date: Sat, 12 Apr 2025 20:39:55 +0300 Subject: [PATCH 3/8] Create script to auto-deploy with retries --- deploy/README.md | 24 +++++++++++---- deploy/docs-ops/default.nix | 1 + deploy/system-ci/default.nix | 27 ++++++++++++++++ deploy/system-physical/default.nix | 27 ++++++++++++++++ deploy/system-vm/default.nix | 27 ++++++++++++++++ deploy/system/default.nix | 20 ------------ dev-shells/default.nix | 2 +- packages/auto-deploy/default.nix | 16 ++++++++++ packages/auto-deploy/script.bash | 49 ++++++++++++++++++++++++++++++ packages/deploy-qois/default.nix | 14 --------- treefmt.nix | 6 ++-- updates.md | 11 ++++--- 12 files changed, 176 insertions(+), 48 deletions(-) create mode 100644 deploy/system-ci/default.nix create mode 100644 deploy/system-physical/default.nix create mode 100644 deploy/system-vm/default.nix delete mode 100644 deploy/system/default.nix create mode 100644 packages/auto-deploy/default.nix create mode 100644 packages/auto-deploy/script.bash delete mode 100644 packages/deploy-qois/default.nix diff --git a/deploy/README.md b/deploy/README.md index 8c95d8a..d0abdbc 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -4,14 +4,26 @@ Note that you have to be connected to the `vpn.qo.is` (or execute the deployment from a host that is in the `backplane.net.qo.is` overlay network) and that you need to have SSH root access to the target machines. +## Deploy system categories + +We currently split out nixosConfigurations into these categories: + +- `system-ci`: Systems should be updated separately because they might break automated deployment processes. +- `system-vm`: Virtual systems. +- `system-physical`: Physical systems. + +You can roll updates with retries by category with: + +```bash +auto-deploy system-vm +auto-deploy system-physical +``` + ## Deploy to selected target hosts ```bash -nix run .#deploy-qois .#.system .#.system -``` +nix develop -## Deploy with extended timeouts (sometimes required for slow APU devices) - -```bash -nix run .#deploy-qois .#calanda.system -- --confirm-timeout 600 --activation-timeout 600 +deploy --skip-checks .#cyprianspitz.system-physical +deploy --skip-checks .#lindberg-build.system-vm ``` diff --git a/deploy/docs-ops/default.nix b/deploy/docs-ops/default.nix index 7fcf9ae..8fd2890 100644 --- a/deploy/docs-ops/default.nix +++ b/deploy/docs-ops/default.nix @@ -12,5 +12,6 @@ in sshUser = "nginx-${domain}"; path = deployPkgs.deploy-rs.lib.activate.noop self.packages.${system}.docs; profilePath = "/var/lib/nginx-${domain}/root"; + remoteBuild = true; }; } diff --git a/deploy/system-ci/default.nix b/deploy/system-ci/default.nix new file mode 100644 index 0000000..a9e49a0 --- /dev/null +++ b/deploy/system-ci/default.nix @@ -0,0 +1,27 @@ +{ + deployPkgs, + pkgs, + self, + ... +}: +let + inherit (pkgs.lib) pipe filterAttrs mapAttrs; +in +{ + nodes = pipe self.nixosConfigurations [ + (filterAttrs (_n: v: v.config.qois.git-ci-runner.enable)) + (mapAttrs ( + host: config: { + hostname = "${host}.backplane.net.qo.is"; + profiles.system-ci = { + sshUser = "root"; + user = "root"; + activationTimeout = 300; + confirmTimeout = 60; + remoteBuild = true; + path = deployPkgs.deploy-rs.lib.activate.nixos config; + }; + } + )) + ]; +} diff --git a/deploy/system-physical/default.nix b/deploy/system-physical/default.nix new file mode 100644 index 0000000..4e60068 --- /dev/null +++ b/deploy/system-physical/default.nix @@ -0,0 +1,27 @@ +{ + deployPkgs, + pkgs, + self, + ... +}: +let + inherit (pkgs.lib) pipe filterAttrs mapAttrs; +in +{ + nodes = pipe self.nixosConfigurations [ + (filterAttrs (_n: v: !v.config.services.qemuGuest.enable && !v.config.qois.git-ci-runner.enable)) + (mapAttrs ( + host: config: { + hostname = "${host}.backplane.net.qo.is"; + profiles.system-physical = { + sshUser = "root"; + user = "root"; + activationTimeout = 600; + confirmTimeout = 120; + remoteBuild = true; + path = deployPkgs.deploy-rs.lib.activate.nixos config; + }; + } + )) + ]; +} diff --git a/deploy/system-vm/default.nix b/deploy/system-vm/default.nix new file mode 100644 index 0000000..65177b4 --- /dev/null +++ b/deploy/system-vm/default.nix @@ -0,0 +1,27 @@ +{ + deployPkgs, + pkgs, + self, + ... +}: +let + inherit (pkgs.lib) pipe filterAttrs mapAttrs; +in +{ + nodes = pipe self.nixosConfigurations [ + (filterAttrs (_n: v: v.config.services.qemuGuest.enable && !v.config.qois.git-ci-runner.enable)) + (mapAttrs ( + host: config: { + hostname = "${host}.backplane.net.qo.is"; + profiles.system-vm = { + sshUser = "root"; + user = "root"; + activationTimeout = 300; + confirmTimeout = 60; + remoteBuild = true; + path = deployPkgs.deploy-rs.lib.activate.nixos config; + }; + } + )) + ]; +} diff --git a/deploy/system/default.nix b/deploy/system/default.nix deleted file mode 100644 index cdaf846..0000000 --- a/deploy/system/default.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ - deployPkgs, - pkgs, - self, - system, - ... -}: -{ - nodes = pkgs.lib.mapAttrs (host: config: { - hostname = "${host}.backplane.net.qo.is"; - profiles.system = { - sshUser = "root"; - user = "root"; - activationTimeout = 420; - confirmTimeout = 120; - - path = deployPkgs.deploy-rs.lib.activate.nixos config; - }; - }) self.nixosConfigurations; -} diff --git a/dev-shells/default.nix b/dev-shells/default.nix index 2023f3e..6f289ff 100644 --- a/dev-shells/default.nix +++ b/dev-shells/default.nix @@ -29,9 +29,9 @@ in pre-commit-check.enabledPackages ++ [ vscodium-with-extensions ] ++ (with self.packages.${system}; [ - deploy-qois sops sops-rekey + auto-deploy ]) ++ (with pkgs; [ attic-client diff --git a/packages/auto-deploy/default.nix b/packages/auto-deploy/default.nix new file mode 100644 index 0000000..3c266a7 --- /dev/null +++ b/packages/auto-deploy/default.nix @@ -0,0 +1,16 @@ +{ + deploy-rs, + gitMinimal, + writeShellApplication, + lib, + ... +}: +writeShellApplication { + name = "auto-deploy"; + meta.description = "Deploy machines automatically."; + runtimeInputs = [ + deploy-rs + gitMinimal + ]; + text = lib.readFile ./script.bash; +} diff --git a/packages/auto-deploy/script.bash b/packages/auto-deploy/script.bash new file mode 100644 index 0000000..66c4520 --- /dev/null +++ b/packages/auto-deploy/script.bash @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +#### Environment +FLAKE_ROOT="$(git rev-parse --show-toplevel)" + +export PROFILE="${1:-''}" +if [ -z "${PROFILE}" ]; then + echo "🛑 Error: No deployment profile was specified as first parameter (e.g. \"${0} system-vm\")" 1>&2 + exit 1 +fi + +HOSTS=$(nix eval --raw "${FLAKE_ROOT}"#deploy.nodes --apply " + nodes: let + inherit (builtins) attrNames filter concatStringsSep; + names = attrNames nodes; + profile = \"${PROFILE}\"; + filteredNames = filter (name: nodes.\${name}.profiles ? \${profile}) names; + in concatStringsSep \"\\n\" filteredNames +") +if [ -z "$HOSTS" ]; then + echo "🛑 Error: No deployments matching the profile ${PROFILE} were found." 1>&2 + exit 1 +fi + +KNOWN_HOSTS_FILE=$(nix eval --raw .#nixosConfigurations.lindberg.config.environment.etc."ssh/ssh_known_hosts".source) + +#### Helpers +retry() { + local -r -i max_attempts="$1" + shift + local -i attempt_num=1 + until "$@"; do + if ((attempt_num == max_attempts)); then + echo "⚠ī¸ Warning: Attempt $attempt_num failed and there are no more attempts left!" + return 1 + else + echo "⚠ī¸ Attempt $attempt_num failed! Trying again in $attempt_num seconds..." + sleep $((attempt_num++)) + fi + done +} + +#### Execution +for HOST in $HOSTS; do + retry 3 deploy \ + --skip-checks \ + --ssh-opts "-o UserKnownHostsFile=${KNOWN_HOSTS_FILE}" \ + --targets "${FLAKE_ROOT}#\"${HOST}\".\"${PROFILE}\"" +done diff --git a/packages/deploy-qois/default.nix b/packages/deploy-qois/default.nix deleted file mode 100644 index 49ab5e7..0000000 --- a/packages/deploy-qois/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ - deploy-rs, - flakeSelf, - writeShellApplication, - ... -}: -writeShellApplication { - name = "deploy-qois"; - meta.description = "Deploy configuration to specificed targets."; - runtimeInputs = [ deploy-rs ]; - text = '' - deploy --remote-build --skip-checks --interactive --targets "''${@:-${flakeSelf}}" - ''; -} diff --git a/treefmt.nix b/treefmt.nix index 5e44926..14412c6 100644 --- a/treefmt.nix +++ b/treefmt.nix @@ -19,11 +19,13 @@ "*.toml" ] ++ [ - ".envrc" - "robots.txt" ".vscode/*" "nixos-modules/system/etc/*" + "private" "private/*" + + ".envrc" + "robots.txt" ]; formatter.jsonfmt.excludes = [ ".vscode/*.json" ]; }; diff --git a/updates.md b/updates.md index 949cac7..4a8f56d 100644 --- a/updates.md +++ b/updates.md @@ -22,13 +22,14 @@ Deploy updates: nix develop # Deploy vms -deploy-qois .#lindberg-nextcloud .#lindberg-build +auto-deploy system-vm -# Deploy fast physical hosts -deploy-qois .#lindberg +# Deploy CI hosts +auto-deploy system-ci + +# Deploy physical hosts +auto-deploy system-physical -# Deploy slow physical hosts (maybe do individually) -deploy-qois --confirm-timeout 600 --activation-timeout 600 --targets .#stompert .#stompert ``` From 1d3201d8e5f385b96094e16424273e46e832bf28 Mon Sep 17 00:00:00 2001 From: Fabian Hauser Date: Sat, 19 Apr 2025 18:11:01 +0300 Subject: [PATCH 4/8] Add SSH_DEPLOY_KEY handling to auto-deploy script --- packages/auto-deploy/script.bash | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/auto-deploy/script.bash b/packages/auto-deploy/script.bash index 66c4520..442eb4c 100644 --- a/packages/auto-deploy/script.bash +++ b/packages/auto-deploy/script.bash @@ -3,12 +3,28 @@ #### Environment FLAKE_ROOT="$(git rev-parse --show-toplevel)" -export PROFILE="${1:-''}" +export PROFILE="${1:-}" if [ -z "${PROFILE}" ]; then echo "🛑 Error: No deployment profile was specified as first parameter (e.g. \"${0} system-vm\")" 1>&2 exit 1 fi +if [ -z "${SSH_DEPLOY_KEY:-}" ]; then + echo "ℹī¸ Info: SSH_DEPLOY_KEY env variable was not set, ignoring." + SSH_KEY_FILE_ARG="" +else + TEMP_KEY_FILE=$(mktemp /dev/shm/ssh_deploy_key.XXXXXXXX) + touch "${TEMP_KEY_FILE}" && chmod 600 "${TEMP_KEY_FILE}" + printf "%s\n" "${SSH_DEPLOY_KEY}" >"${TEMP_KEY_FILE}" + SSH_KEY_FILE_ARG="-i ${TEMP_KEY_FILE}" + + # Set up a trap to remove the temporary key file on script exit + trap 'rm -f "${TEMP_KEY_FILE}"' EXIT + trap 'rm -f "${TEMP_KEY_FILE}"' SIGINT + trap 'rm -f "${TEMP_KEY_FILE}"' SIGTERM + trap 'rm -f "${TEMP_KEY_FILE}"' SIGQUIT +fi + HOSTS=$(nix eval --raw "${FLAKE_ROOT}"#deploy.nodes --apply " nodes: let inherit (builtins) attrNames filter concatStringsSep; @@ -31,7 +47,7 @@ retry() { local -i attempt_num=1 until "$@"; do if ((attempt_num == max_attempts)); then - echo "⚠ī¸ Warning: Attempt $attempt_num failed and there are no more attempts left!" + echo "🛑 Error: Attempt $attempt_num failed and there are no more attempts left!" 1>&2 return 1 else echo "⚠ī¸ Attempt $attempt_num failed! Trying again in $attempt_num seconds..." @@ -44,6 +60,6 @@ retry() { for HOST in $HOSTS; do retry 3 deploy \ --skip-checks \ - --ssh-opts "-o UserKnownHostsFile=${KNOWN_HOSTS_FILE}" \ + --ssh-opts "-o UserKnownHostsFile=${KNOWN_HOSTS_FILE} ${SSH_KEY_FILE_ARG:-}" \ --targets "${FLAKE_ROOT}#\"${HOST}\".\"${PROFILE}\"" done From 54c4cf23ff9614633f5b5baf852f35cdc7b29563 Mon Sep 17 00:00:00 2001 From: Fabian Hauser Date: Sat, 19 Apr 2025 18:20:30 +0300 Subject: [PATCH 5/8] Update CI pipleline for auto deployment --- .github/workflows/ci.yml | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c40fdf4..0d886cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,11 +25,22 @@ jobs: attic use "$CACHE_REPOSITORY" - name: Run Builds and Checks run: nix-fast-build --no-nom --max-jobs 6 --skip-cached --attic-cache "$CACHE_REPOSITORY" - - name: Deploy Docs - if: success() && github.ref == 'refs/heads/main' - run: | - mkdir ~/.ssh/ - echo -e "Host lindberg-webapps.backplane.net.qo.is\n StrictHostKeyChecking no" >> ~/.ssh/config - (umask 0077 && printf "%s\n" "${{ secrets.SSH_DEPLOY_KEY }}" > ~/.ssh/id_ed25519) - deploy --skip-checks --remote-build .#lindberg-webapps.\"docs-ops.qo.is\" - # Remote build is neccessary due to non-wheel nix users signing restrictions. However, the build should come from the cache anyway. + deploy: + needs: build + if: success() && github.ref == 'refs/heads/main' + runs-on: nix + env: + SSH_DEPLOY_KEY: "${{ secrets.SSH_DEPLOY_KEY }}" + strategy: + matrix: + profile: + - docs-ops.qo.is + - system-vm + steps: + - name: Initialize CI + uses: https://git.qo.is/qo.is/actions-nix-init@main + with: + token: ${{ secrets.CI_TOKEN }} + lfs: false + - name: "Deploy profile" + run: "auto-deploy ${{ matrix.profile }}" From 99a5abbbfe522e89cc6edd391df707251f9cc95b Mon Sep 17 00:00:00 2001 From: Fabian Hauser Date: Sat, 19 Apr 2025 18:42:34 +0300 Subject: [PATCH 6/8] Add deployment ssh key to all VMs --- nixos-modules/system/virtual-machine.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nixos-modules/system/virtual-machine.nix b/nixos-modules/system/virtual-machine.nix index 5e5a8ae..776571e 100644 --- a/nixos-modules/system/virtual-machine.nix +++ b/nixos-modules/system/virtual-machine.nix @@ -13,6 +13,10 @@ with lib; config = lib.mkIf cfg.enable { + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBS65v7n5ozOUjYGuO/dgLC9C5MUGL5kTnQnvWAYP5B3 ci@git.qo.is" + ]; # TODO: Move this key to allow CI deployment for all machines. + boot.loader.grub.enable = true; system.autoUpgrade.allowReboot = true; From 2c679066ef88aabf37df08dca3f1e0bb66898c24 Mon Sep 17 00:00:00 2001 From: Fabian Hauser Date: Sat, 19 Apr 2025 18:57:37 +0300 Subject: [PATCH 7/8] Set rennovate schedule to run automatically daily. --- renovate.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/renovate.json b/renovate.json index f710e2d..63404e8 100644 --- a/renovate.json +++ b/renovate.json @@ -3,11 +3,12 @@ "extends": [ "config:recommended" ], + "schedule": [ + "* 18-19 * * *" + ], "lockFileMaintenance": { "enabled": true, - "extends": [ - "schedule:weekly" - ] + "automerge": true }, "cloneSubmodules": true, "nix": { From 7208e2f4ba6295cfa07fd2469a26e28d51c35c25 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sat, 19 Apr 2025 18:00:59 +0200 Subject: [PATCH 8/8] chore(deps): lock file maintenance --- flake.lock | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/flake.lock b/flake.lock index 9af33a5..a23e572 100644 --- a/flake.lock +++ b/flake.lock @@ -27,11 +27,11 @@ ] }, "locked": { - "lastModified": 1741786315, - "narHash": "sha256-VT65AE2syHVj6v/DGB496bqBnu1PXrrzwlw07/Zpllc=", + "lastModified": 1744940522, + "narHash": "sha256-TNoetfICvd29DhxRPpmyKItQBDlqSvKcV+wGNkn14jk=", "owner": "nix-community", "repo": "disko", - "rev": "0d8c6ad4a43906d14abd5c60e0ffe7b587b213de", + "rev": "51d33bbb7f1e74ba5f9d9a77357735149da99081", "type": "github" }, "original": { @@ -133,11 +133,11 @@ }, "nixpkgs-nixos-stable": { "locked": { - "lastModified": 1743231893, - "narHash": "sha256-tpJsHMUPEhEnzySoQxx7+kA+KUtgWqvlcUBqROYNNt0=", + "lastModified": 1744440957, + "narHash": "sha256-FHlSkNqFmPxPJvy+6fNLaNeWnF1lZSgqVCl/eWaJRc4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c570c1f5304493cafe133b8d843c7c1c4a10d3a6", + "rev": "26d499fc9f1d567283d5d56fcf367edd815dba1d", "type": "github" }, "original": { @@ -149,11 +149,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1743095683, - "narHash": "sha256-gWd4urRoLRe8GLVC/3rYRae1h+xfQzt09xOfb0PaHSk=", + "lastModified": 1744932701, + "narHash": "sha256-fusHbZCyv126cyArUwwKrLdCkgVAIaa/fQJYFlCEqiU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5e5402ecbcb27af32284d4a62553c019a3a49ea6", + "rev": "b024ced1aac25639f8ca8fdfc2f8c4fbd66c48ef", "type": "github" }, "original": { @@ -172,7 +172,6 @@ "locked": { "lastModified": 1743779182, "narHash": "sha256-0wQ+22g6dfnIBIvw2Mji37R7OmCMrmqcp/4zSLQRKSQ=", - "ref": "refs/heads/main", "rev": "8e7ec0d7f4a571a55d0e6b99fcd6f85fea474f1e", "revCount": 16, "type": "git", @@ -202,11 +201,11 @@ ] }, "locked": { - "lastModified": 1743305778, - "narHash": "sha256-Ux/UohNtnM5mn9SFjaHp6IZe2aAnUCzklMluNtV6zFo=", + "lastModified": 1744669848, + "narHash": "sha256-pXyanHLUzLNd3MX9vsWG+6Z2hTU8niyphWstYEP3/GU=", "owner": "Mic92", "repo": "sops-nix", - "rev": "8e873886bbfc32163fe027b8676c75637b7da114", + "rev": "61154300d945f0b147b30d24ddcafa159148026a", "type": "github" }, "original": { @@ -237,11 +236,11 @@ ] }, "locked": { - "lastModified": 1743081648, - "narHash": "sha256-WRAylyYptt6OX5eCEBWyTwOEqEtD6zt33rlUkr6u3cE=", + "lastModified": 1744961264, + "narHash": "sha256-aRmUh0AMwcbdjJHnytg1e5h5ECcaWtIFQa6d9gI85AI=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "29a3d7b768c70addce17af0869f6e2bd8f5be4b7", + "rev": "8d404a69efe76146368885110f29a2ca3700bee6", "type": "github" }, "original": {