From 6bbd0f0157f88d01130e87e889aaca1d142b7169 Mon Sep 17 00:00:00 2001 From: Fabian Hauser Date: Thu, 3 Oct 2024 13:46:37 +0300 Subject: [PATCH] Use specific git forgejo CI --- flake.lock | 8 +- .../lindberg-build/applications/default.nix | 1 - .../applications/gitlab-runner.nix | 27 -- nixos-modules/qois/git-ci-runner/default.nix | 282 ++++++++++++++++-- private | 2 +- 5 files changed, 260 insertions(+), 60 deletions(-) delete mode 100644 nixos-configurations/lindberg-build/applications/gitlab-runner.nix diff --git a/flake.lock b/flake.lock index b5e6a6c..8b99d53 100644 --- a/flake.lock +++ b/flake.lock @@ -223,11 +223,11 @@ ] }, "locked": { - "lastModified": 1727557927, - "narHash": "sha256-+dTv85ZXAatKiCu5VKTQkFE/RmWdlXwkuPvjOmfcPBI=", + "lastModified": 1727954097, + "narHash": "sha256-Fmi1bGcyVLVMpSURwXnGCwWl5K0MVAJHuybDa/vYDis=", "ref": "refs/heads/main", - "rev": "9a646336c5ad419ec79ae81a47d68213bdcbff92", - "revCount": 5, + "rev": "1d096ecce6a9b722dbdc70515375ec6798958c23", + "revCount": 6, "type": "git", "url": "file:./private" }, diff --git a/nixos-configurations/lindberg-build/applications/default.nix b/nixos-configurations/lindberg-build/applications/default.nix index 7a0aa4c..d634828 100644 --- a/nixos-configurations/lindberg-build/applications/default.nix +++ b/nixos-configurations/lindberg-build/applications/default.nix @@ -2,7 +2,6 @@ { imports = [ - ./gitlab-runner.nix ./attic.nix ./nixpkgs-cache.nix ]; diff --git a/nixos-configurations/lindberg-build/applications/gitlab-runner.nix b/nixos-configurations/lindberg-build/applications/gitlab-runner.nix deleted file mode 100644 index cf68c3a..0000000 --- a/nixos-configurations/lindberg-build/applications/gitlab-runner.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ config, pkgs, ... }: -{ - - services.gitlab-runner = { - enable = true; - - gracefulTimeout = "20min"; - - clear-docker-cache = { - enable = true; - dates = "monthly"; - }; - - services = { - default = { - runUntagged = true; - # File should contain at least these two variables: - # `CI_SERVER_URL` - # `REGISTRATION_TOKEN` - registrationConfigFile = config.sops.secrets."gitlab-runner/default-registration".path; - dockerImage = "debian:stable"; - limit = 42; # The magic value - maximumTimeout = 7200; # 2h oughta be enough for everyone - }; - }; - }; -} diff --git a/nixos-modules/qois/git-ci-runner/default.nix b/nixos-modules/qois/git-ci-runner/default.nix index 3efae16..4d02566 100644 --- a/nixos-modules/qois/git-ci-runner/default.nix +++ b/nixos-modules/qois/git-ci-runner/default.nix @@ -19,34 +19,262 @@ with lib; default = "git.qo.is"; description = "Domain, under which the service is served."; }; - }; - config = mkIf cfg.enable { - - sops.secrets."forgejo/runner-token/${defaultInstanceName}".restartUnits = [ - "gitea-runner-${defaultInstanceName}.service" - ]; - - services.gitea-actions-runner = { - package = pkgs.forgejo-runner; - instances.${defaultInstanceName} = { - enable = true; - name = "${config.networking.hostName}-${defaultInstanceName}"; - url = "https://${cfg.domain}"; - tokenFile = config.sops.secrets."forgejo/runner-token/${defaultInstanceName}".path; - labels = [ - "ubuntu-latest:docker://gitea/runner-images:ubuntu-latest" - "ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04" - "docker:docker://code.forgejo.org/oci/alpine:3.20" - ]; - settings = { - log.level = "warn"; - runner = { - capacity = 30; - }; - cache.enable = true; # TODO: This should probably be a central cache server? - }; - }; + nixInstances = mkOption { + type = types.numbers.positive; + default = 1; + description = "How many nix runner instances to start"; }; }; + + config = mkIf cfg.enable (mkMerge [ + { + + sops.secrets."forgejo/runner-registration-token".restartUnits = [ + "gitea-runner-${defaultInstanceName}.service" + ]; + + virtualisation.podman = { + enable = true; + dockerCompat = true; + dockerSocket.enable = true; + autoPrune.enable = true; + }; + + services.gitea-actions-runner = { + package = pkgs.forgejo-runner; + instances.${defaultInstanceName} = { + enable = true; + name = "${config.networking.hostName}-${defaultInstanceName}"; + url = "https://${cfg.domain}"; + tokenFile = config.sops.secrets."forgejo/runner-registration-token".path; + labels = [ + "ubuntu-latest:docker://gitea/runner-images:ubuntu-latest" + "ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04" + "docker:docker://code.forgejo.org/oci/alpine:3.20" + ]; + settings = { + log.level = "warn"; + runner = { + capacity = 30; + }; + cache.enable = true; # TODO: This should probably be a central cache server? + }; + }; + }; + } + + { + # everything here has no dependencies on the store + systemd.services.gitea-runner-nix-image = { + wantedBy = [ "multi-user.target" ]; + after = [ "podman.service" ]; + requires = [ "podman.service" ]; + path = [ + config.virtualisation.podman.package + pkgs.gnutar + pkgs.shadow + pkgs.getent + ]; + # we also include etc here because the cleanup job also wants the nixuser to be present + script = '' + set -eux -o pipefail + mkdir -p etc/nix + + # Create an unpriveleged user that we can use also without the run-as-user.sh script + touch etc/passwd etc/group + groupid=$(cut -d: -f3 < <(getent group nixuser)) + userid=$(cut -d: -f3 < <(getent passwd nixuser)) + groupadd --prefix $(pwd) --gid "$groupid" nixuser + emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.' + useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser + + cat < etc/nix/nix.conf + accept-flake-config = true + experimental-features = nix-command flakes + NIX_CONFIG + + cat < etc/nsswitch.conf + passwd: files mymachines systemd + group: files mymachines systemd + shadow: files + + hosts: files mymachines dns myhostname + networks: files + + ethers: files + services: files + protocols: files + rpc: files + NSSWITCH + + # list the content as it will be imported into the container + tar -cv . | tar -tvf - + tar -cv . | podman import - gitea-runner-nix + ''; + serviceConfig = { + RuntimeDirectory = "gitea-runner-nix-image"; + WorkingDirectory = "/run/gitea-runner-nix-image"; + Type = "oneshot"; + RemainAfterExit = true; + }; + }; + + users.users.nixuser = { + group = "nixuser"; + description = "Used for running nix ci jobs"; + home = "/var/empty"; + isSystemUser = true; + }; + users.groups.nixuser = { }; + } + { + virtualisation = { + podman.enable = true; + }; + + virtualisation.containers.storage.settings = { + storage.driver = "btrfs"; + storage.graphroot = "/var/lib/containers/storage"; + storage.runroot = "/run/containers/storage"; + }; + + virtualisation.containers.containersConf.settings = { + # podman seems to not work with systemd-resolved + containers.dns_servers = [ + "8.8.8.8" + "8.8.4.4" + ]; + }; + } + { + systemd.services = + genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}") cfg.nixInstances) + (name: { + after = [ + "gitea-runner-nix-image.service" + ]; + requires = [ + "gitea-runner-nix-image.service" + ]; + + # TODO: systemd confinment + serviceConfig = { + # Hardening (may overlap with DynamicUser=) + # The following options are only for optimizing output of systemd-analyze + AmbientCapabilities = ""; + CapabilityBoundingSet = ""; + # ProtectClock= adds DeviceAllow=char-rtc r + DeviceAllow = ""; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + PrivateUsers = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + UMask = "0066"; + ProtectProc = "invisible"; + SystemCallFilter = [ + "~@clock" + "~@cpu-emulation" + "~@module" + "~@mount" + "~@obsolete" + "~@raw-io" + "~@reboot" + "~@swap" + # needed by go? + #"~@resources" + "~@privileged" + "~capset" + "~setdomainname" + "~sethostname" + ]; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + "AF_NETLINK" + ]; + + # Needs network access + PrivateNetwork = false; + # Cannot be true due to Node + MemoryDenyWriteExecute = false; + + # The more restrictive "pid" option makes `nix` commands in CI emit + # "GC Warning: Couldn't read /proc/stat" + # You may want to set this to "pid" if not using `nix` commands + ProcSubset = "all"; + # Coverage programs for compiled code such as `cargo-tarpaulin` disable + # ASLR (address space layout randomization) which requires the + # `personality` syscall + # You may want to set this to `true` if not using coverage tooling on + # compiled code + LockPersonality = false; + + # Note that this has some interactions with the User setting; so you may + # want to consult the systemd docs if using both. + DynamicUser = true; + }; + }); + + services.gitea-actions-runner.instances = + let + storeDeps = pkgs.runCommand "store-deps" { } '' + mkdir -p $out/bin + for dir in ${ + toString [ + pkgs.coreutils + pkgs.findutils + pkgs.gnugrep + pkgs.gawk + pkgs.git + pkgs.nix + pkgs.bash + pkgs.jq + pkgs.nodejs + ] + }; do + for bin in "$dir"/bin/*; do + ln -s "$bin" "$out/bin/$(basename "$bin")" + done + done + + # Add SSL CA certs + mkdir -p $out/etc/ssl/certs + cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt + ''; + in + genAttrs (builtins.genList (n: "nix${builtins.toString n}") cfg.nixInstances) (name: { + enable = true; + name = "${config.networking.hostName}-${name}"; + url = "https://${cfg.domain}"; + tokenFile = config.sops.secrets."forgejo/runner-registration-token".path; + labels = [ "nix:docker://gitea-runner-nix" ]; + settings = { + container.options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nixuser --device=/dev/kvm"; + # the default network that also respects our dns server settings + container.network = "host"; + container.valid_volumes = [ + "/nix" + "${storeDeps}/bin" + "${storeDeps}/etc/ssl" + ]; + }; + }); + } + ]); + } diff --git a/private b/private index 9a64633..1d096ec 160000 --- a/private +++ b/private @@ -1 +1 @@ -Subproject commit 9a646336c5ad419ec79ae81a47d68213bdcbff92 +Subproject commit 1d096ecce6a9b722dbdc70515375ec6798958c23