Use specific git forgejo CI

flake.lock

@@ -223,11 +223,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1727557927,
+        "lastModified": 1727954097,
-        "narHash": "sha256-+dTv85ZXAatKiCu5VKTQkFE/RmWdlXwkuPvjOmfcPBI=",
+        "narHash": "sha256-Fmi1bGcyVLVMpSURwXnGCwWl5K0MVAJHuybDa/vYDis=",
         "ref": "refs/heads/main",
-        "rev": "9a646336c5ad419ec79ae81a47d68213bdcbff92",
+        "rev": "1d096ecce6a9b722dbdc70515375ec6798958c23",
-        "revCount": 5,
+        "revCount": 6,
         "type": "git",
         "url": "file:./private"
       },

nixos-configurations/lindberg-build/applications/default.nix

@@ -2,7 +2,6 @@
 {
 
   imports = [
-    ./gitlab-runner.nix
     ./attic.nix
     ./nixpkgs-cache.nix
   ];

nixos-configurations/lindberg-build/applications/gitlab-runner.nix

@@ -1,27 +0,0 @@
-{ config, pkgs, ... }:
-{
-
-  services.gitlab-runner = {
-    enable = true;
-
-    gracefulTimeout = "20min";
-
-    clear-docker-cache = {
-      enable = true;
-      dates = "monthly";
-    };
-
-    services = {
-      default = {
-        runUntagged = true;
-        # File should contain at least these two variables:
-        # `CI_SERVER_URL`
-        # `REGISTRATION_TOKEN`
-        registrationConfigFile = config.sops.secrets."gitlab-runner/default-registration".path;
-        dockerImage = "debian:stable";
-        limit = 42; # The magic value
-        maximumTimeout = 7200; # 2h oughta be enough for everyone
-      };
-    };
-  };
-}

nixos-modules/qois/git-ci-runner/default.nix

@@ -19,34 +19,262 @@ with lib;
       default = "git.qo.is";
       description = "Domain, under which the service is served.";
     };
-  };
 
-  config = mkIf cfg.enable {
+    nixInstances = mkOption {
-
+      type = types.numbers.positive;
-    sops.secrets."forgejo/runner-token/${defaultInstanceName}".restartUnits = [
+      default = 1;
-      "gitea-runner-${defaultInstanceName}.service"
+      description = "How many nix runner instances to start";
-    ];
-
-    services.gitea-actions-runner = {
-      package = pkgs.forgejo-runner;
-      instances.${defaultInstanceName} = {
-        enable = true;
-        name = "${config.networking.hostName}-${defaultInstanceName}";
-        url = "https://${cfg.domain}";
-        tokenFile = config.sops.secrets."forgejo/runner-token/${defaultInstanceName}".path;
-        labels = [
-          "ubuntu-latest:docker://gitea/runner-images:ubuntu-latest"
-          "ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04"
-          "docker:docker://code.forgejo.org/oci/alpine:3.20"
-        ];
-        settings = {
-          log.level = "warn";
-          runner = {
-            capacity = 30;
-          };
-          cache.enable = true; # TODO: This should probably be a central cache server?
-        };
-      };
     };
   };
+
+  config = mkIf cfg.enable (mkMerge [
+    {
+
+      sops.secrets."forgejo/runner-registration-token".restartUnits = [
+        "gitea-runner-${defaultInstanceName}.service"
+      ];
+
+      virtualisation.podman = {
+        enable = true;
+        dockerCompat = true;
+        dockerSocket.enable = true;
+        autoPrune.enable = true;
+      };
+
+      services.gitea-actions-runner = {
+        package = pkgs.forgejo-runner;
+        instances.${defaultInstanceName} = {
+          enable = true;
+          name = "${config.networking.hostName}-${defaultInstanceName}";
+          url = "https://${cfg.domain}";
+          tokenFile = config.sops.secrets."forgejo/runner-registration-token".path;
+          labels = [
+            "ubuntu-latest:docker://gitea/runner-images:ubuntu-latest"
+            "ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04"
+            "docker:docker://code.forgejo.org/oci/alpine:3.20"
+          ];
+          settings = {
+            log.level = "warn";
+            runner = {
+              capacity = 30;
+            };
+            cache.enable = true; # TODO: This should probably be a central cache server?
+          };
+        };
+      };
+    }
+
+    {
+      # everything here has no dependencies on the store
+      systemd.services.gitea-runner-nix-image = {
+        wantedBy = [ "multi-user.target" ];
+        after = [ "podman.service" ];
+        requires = [ "podman.service" ];
+        path = [
+          config.virtualisation.podman.package
+          pkgs.gnutar
+          pkgs.shadow
+          pkgs.getent
+        ];
+        # we also include etc here because the cleanup job also wants the nixuser to be present
+        script = ''
+          set -eux -o pipefail
+          mkdir -p etc/nix
+
+          # Create an unpriveleged user that we can use also without the run-as-user.sh script
+          touch etc/passwd etc/group
+          groupid=$(cut -d: -f3 < <(getent group nixuser))
+          userid=$(cut -d: -f3 < <(getent passwd nixuser))
+          groupadd --prefix $(pwd) --gid "$groupid" nixuser
+          emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
+          useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser
+
+          cat <<NIX_CONFIG > etc/nix/nix.conf
+          accept-flake-config = true
+          experimental-features = nix-command flakes
+          NIX_CONFIG
+
+          cat <<NSSWITCH > etc/nsswitch.conf
+          passwd:    files mymachines systemd
+          group:     files mymachines systemd
+          shadow:    files
+
+          hosts:     files mymachines dns myhostname
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+          NSSWITCH
+
+          # list the content as it will be imported into the container
+          tar -cv . | tar -tvf -
+          tar -cv . | podman import - gitea-runner-nix
+        '';
+        serviceConfig = {
+          RuntimeDirectory = "gitea-runner-nix-image";
+          WorkingDirectory = "/run/gitea-runner-nix-image";
+          Type = "oneshot";
+          RemainAfterExit = true;
+        };
+      };
+
+      users.users.nixuser = {
+        group = "nixuser";
+        description = "Used for running nix ci jobs";
+        home = "/var/empty";
+        isSystemUser = true;
+      };
+      users.groups.nixuser = { };
+    }
+    {
+      virtualisation = {
+        podman.enable = true;
+      };
+
+      virtualisation.containers.storage.settings = {
+        storage.driver = "btrfs";
+        storage.graphroot = "/var/lib/containers/storage";
+        storage.runroot = "/run/containers/storage";
+      };
+
+      virtualisation.containers.containersConf.settings = {
+        # podman seems to not work with systemd-resolved
+        containers.dns_servers = [
+          "8.8.8.8"
+          "8.8.4.4"
+        ];
+      };
+    }
+    {
+      systemd.services =
+        genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}") cfg.nixInstances)
+          (name: {
+            after = [
+              "gitea-runner-nix-image.service"
+            ];
+            requires = [
+              "gitea-runner-nix-image.service"
+            ];
+
+            # TODO: systemd confinment
+            serviceConfig = {
+              # Hardening (may overlap with DynamicUser=)
+              # The following options are only for optimizing output of systemd-analyze
+              AmbientCapabilities = "";
+              CapabilityBoundingSet = "";
+              # ProtectClock= adds DeviceAllow=char-rtc r
+              DeviceAllow = "";
+              NoNewPrivileges = true;
+              PrivateDevices = true;
+              PrivateMounts = true;
+              PrivateTmp = true;
+              PrivateUsers = true;
+              ProtectClock = true;
+              ProtectControlGroups = true;
+              ProtectHome = true;
+              ProtectHostname = true;
+              ProtectKernelLogs = true;
+              ProtectKernelModules = true;
+              ProtectKernelTunables = true;
+              ProtectSystem = "strict";
+              RemoveIPC = true;
+              RestrictNamespaces = true;
+              RestrictRealtime = true;
+              RestrictSUIDSGID = true;
+              UMask = "0066";
+              ProtectProc = "invisible";
+              SystemCallFilter = [
+                "~@clock"
+                "~@cpu-emulation"
+                "~@module"
+                "~@mount"
+                "~@obsolete"
+                "~@raw-io"
+                "~@reboot"
+                "~@swap"
+                # needed by go?
+                #"~@resources"
+                "~@privileged"
+                "~capset"
+                "~setdomainname"
+                "~sethostname"
+              ];
+              RestrictAddressFamilies = [
+                "AF_INET"
+                "AF_INET6"
+                "AF_UNIX"
+                "AF_NETLINK"
+              ];
+
+              # Needs network access
+              PrivateNetwork = false;
+              # Cannot be true due to Node
+              MemoryDenyWriteExecute = false;
+
+              # The more restrictive "pid" option makes `nix` commands in CI emit
+              # "GC Warning: Couldn't read /proc/stat"
+              # You may want to set this to "pid" if not using `nix` commands
+              ProcSubset = "all";
+              # Coverage programs for compiled code such as `cargo-tarpaulin` disable
+              # ASLR (address space layout randomization) which requires the
+              # `personality` syscall
+              # You may want to set this to `true` if not using coverage tooling on
+              # compiled code
+              LockPersonality = false;
+
+              # Note that this has some interactions with the User setting; so you may
+              # want to consult the systemd docs if using both.
+              DynamicUser = true;
+            };
+          });
+
+      services.gitea-actions-runner.instances =
+        let
+          storeDeps = pkgs.runCommand "store-deps" { } ''
+            mkdir -p $out/bin
+            for dir in ${
+              toString [
+                pkgs.coreutils
+                pkgs.findutils
+                pkgs.gnugrep
+                pkgs.gawk
+                pkgs.git
+                pkgs.nix
+                pkgs.bash
+                pkgs.jq
+                pkgs.nodejs
+              ]
+            }; do
+              for bin in "$dir"/bin/*; do
+                ln -s "$bin" "$out/bin/$(basename "$bin")"
+              done
+            done
+
+            # Add SSL CA certs
+            mkdir -p $out/etc/ssl/certs
+            cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt
+          '';
+        in
+        genAttrs (builtins.genList (n: "nix${builtins.toString n}") cfg.nixInstances) (name: {
+          enable = true;
+          name = "${config.networking.hostName}-${name}";
+          url = "https://${cfg.domain}";
+          tokenFile = config.sops.secrets."forgejo/runner-registration-token".path;
+          labels = [ "nix:docker://gitea-runner-nix" ];
+          settings = {
+            container.options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nixuser --device=/dev/kvm";
+            # the default network that also respects our dns server settings
+            container.network = "host";
+            container.valid_volumes = [
+              "/nix"
+              "${storeDeps}/bin"
+              "${storeDeps}/etc/ssl"
+            ];
+          };
+        });
+    }
+  ]);
+
 }

private

@@ -1 +1 @@
-Subproject commit 9a646336c5ad419ec79ae81a47d68213bdcbff92
+Subproject commit 1d096ecce6a9b722dbdc70515375ec6798958c23