nixos/darwin-builder: add disk space options (#224480)

2 files changed, 263 insertions, 117 deletions

diff --git a/doc/builders/special/darwin-builder.section.md b/doc/builders/special/darwin-builder.section.md
index 3913b692498d..30bf2d095102 100644
--- a/doc/builders/special/darwin-builder.section.md
+++ b/doc/builders/special/darwin-builder.section.md
@@ -61,3 +61,89 @@ builders-use-substitutes = true
 ```ShellSession
 $ sudo launchctl kickstart -k system/org.nixos.nix-daemon
 ```
+
+## Example flake usage
+
+```
+{
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-22.11-darwin";
+    darwin.url = "github:lnl7/nix-darwin/master";
+    darwin.inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = { self, darwin, nixpkgs, ... }@inputs:
+  let
+
+    inherit (darwin.lib) darwinSystem;
+    system = "aarch64-darwin";
+    pkgs = nixpkgs.legacyPackages."${system}";
+    linuxSystem = builtins.replaceStrings [ "darwin" ] [ "linux" ] system;
+
+    darwin-builder = nixpkgs.lib.nixosSystem {
+      system = linuxSystem;
+      modules = [
+        "${nixpkgs}/nixos/modules/profiles/macos-builder.nix"
+        { virtualisation.host.pkgs = pkgs; }
+      ];
+    };
+  in {
+
+    darwinConfigurations = {
+      machine1 = darwinSystem {
+        inherit system;
+        modules = [
+          {
+            nix.distributedBuilds = true;
+            nix.buildMachines = [{
+              hostName = "ssh://builder@localhost";
+              system = linuxSystem;
+              maxJobs = 4;
+              supportedFeatures = [ "kvm" "benchmark" "big-parallel" ];
+            }];
+
+            launchd.daemons.darwin-builder = {
+              command = "${darwin-builder.config.system.build.macos-builder-installer}/bin/create-builder";
+              serviceConfig = {
+                KeepAlive = true;
+                RunAtLoad = true;
+                StandardOutPath = "/var/log/darwin-builder.log";
+                StandardErrorPath = "/var/log/darwin-builder.log";
+              };
+            };
+          }
+        ];
+      };
+    };
+
+  };
+}
+```
+
+## Reconfiguring the builder
+
+Initially you should not change the builder configuration else you will not be
+able to use the binary cache. However, after you have the builder running locally
+you may use it to build a modified builder with additional storage or memory.
+
+To do this, you just need to set the `virtualisation.darwin-builder.*` parameters as
+in the example below and rebuild.
+
+```
+    darwin-builder = nixpkgs.lib.nixosSystem {
+      system = linuxSystem;
+      modules = [
+        "${nixpkgs}/nixos/modules/profiles/macos-builder.nix"
+        {
+          virtualisation.host.pkgs = pkgs;
+          virtualisation.darwin-builder.diskSize = 5120;
+          virtualisation.darwin-builder.memorySize = 1024;
+          virtualisation.darwin-builder.hostPort = 33022;
+          virtualisation.darwin-builder.workingDirectory = "/var/lib/darwin-builder";
+        }
+      ];
+```
+
+You may make any other changes to your VM in this attribute set. For example,
+you could enable Docker or X11 forwarding to your Darwin host.
+
diff --git a/nixos/modules/profiles/macos-builder.nix b/nixos/modules/profiles/macos-builder.nix
index 4a5359582bce..768c673e7f37 100644
--- a/nixos/modules/profiles/macos-builder.nix
+++ b/nixos/modules/profiles/macos-builder.nix
@@ -7,6 +7,8 @@ let
 
   keyType = "ed25519";
 
+  cfg = config.virtualisation.darwin-builder;
+
 in
 
 {
@@ -24,156 +26,214 @@ in
     }
   ];
 
-  # The builder is not intended to be used interactively
-  documentation.enable = false;
+  options.virtualisation.darwin-builder = with lib; {
+    diskSize = mkOption {
+      default = 20 * 1024;
+      type = types.int;
+      example = 30720;
+      description = "The maximum disk space allocated to the runner in MB";
+    };
+    memorySize = mkOption {
+      default = 3 * 1024;
+      type = types.int;
+      example = 8192;
+      description = "The runner's memory in MB";
+    };
+    min-free = mkOption {
+      default = 1024 * 1024 * 1024;
+      type = types.int;
+      example = 1073741824;
+      description = ''
+        The threshold (in bytes) of free disk space left at which to
+        start garbage collection on the runner
+      '';
+    };
+    max-free = mkOption {
+      default = 3 * 1024 * 1024 * 1024;
+      type = types.int;
+      example = 3221225472;
+      description = ''
+        The threshold (in bytes) of free disk space left at which to
+        stop garbage collection on the runner
+      '';
+    };
+    workingDirectory = mkOption {
+       default = ".";
+       type = types.str;
+       example = "/var/lib/darwin-builder";
+       description = ''
+         The working directory to use to run the script. When running
+         as part of a flake will need to be set to a non read-only filesystem.
+       '';
+    };
+    hostPort = mkOption {
+      default = 22;
+      type = types.int;
+      example = 31022;
+      description = ''
+        The localhost host port to forward TCP to the guest port.
+      '';
+    };
+  };
+
+  config = {
+    # The builder is not intended to be used interactively
+    documentation.enable = false;
 
-  environment.etc = {
-    "ssh/ssh_host_ed25519_key" = {
-      mode = "0600";
+    environment.etc = {
+      "ssh/ssh_host_ed25519_key" = {
+        mode = "0600";
 
-      source = ./keys/ssh_host_ed25519_key;
-    };
+        source = ./keys/ssh_host_ed25519_key;
+      };
 
-    "ssh/ssh_host_ed25519_key.pub" = {
-      mode = "0644";
+      "ssh/ssh_host_ed25519_key.pub" = {
+        mode = "0644";
 
-      source = ./keys/ssh_host_ed25519_key.pub;
+        source = ./keys/ssh_host_ed25519_key.pub;
+      };
     };
-  };
 
-  # DNS fails for QEMU user networking (SLiRP) on macOS.  See:
-  #
-  # https://github.com/utmapp/UTM/issues/2353
-  #
-  # This works around that by using a public DNS server other than the DNS
-  # server that QEMU provides (normally 10.0.2.3)
-  networking.nameservers = [ "8.8.8.8" ];
+    # DNS fails for QEMU user networking (SLiRP) on macOS.  See:
+    #
+    # https://github.com/utmapp/UTM/issues/2353
+    #
+    # This works around that by using a public DNS server other than the DNS
+    # server that QEMU provides (normally 10.0.2.3)
+    networking.nameservers = [ "8.8.8.8" ];
 
-  nix.settings = {
-    auto-optimise-store = true;
+    nix.settings = {
+      auto-optimise-store = true;
 
-    min-free = 1024 * 1024 * 1024;
+      min-free = cfg.min-free;
 
-    max-free = 3 * 1024 * 1024 * 1024;
+      max-free = cfg.max-free;
 
-    trusted-users = [ "root" user ];
-  };
+      trusted-users = [ "root" user ];
+    };
 
-  services = {
-    getty.autologinUser = user;
+    services = {
+      getty.autologinUser = user;
 
-    openssh = {
-      enable = true;
+      openssh = {
+        enable = true;
 
-      authorizedKeysFiles = [ "${keysDirectory}/%u_${keyType}.pub" ];
+        authorizedKeysFiles = [ "${keysDirectory}/%u_${keyType}.pub" ];
+      };
     };
-  };
 
-  system.build.macos-builder-installer =
-    let
-      privateKey = "/etc/nix/${user}_${keyType}";
+    system.build.macos-builder-installer =
+      let
+        privateKey = "/etc/nix/${user}_${keyType}";
 
-      publicKey = "${privateKey}.pub";
+        publicKey = "${privateKey}.pub";
 
-      # This installCredentials script is written so that it's as easy as
-      # possible for a user to audit before confirming the `sudo`
-      installCredentials = hostPkgs.writeShellScript "install-credentials" ''
-        KEYS="''${1}"
-        INSTALL=${hostPkgs.coreutils}/bin/install
-        "''${INSTALL}" -g nixbld -m 600 "''${KEYS}/${user}_${keyType}" ${privateKey}
-        "''${INSTALL}" -g nixbld -m 644 "''${KEYS}/${user}_${keyType}.pub" ${publicKey}
-      '';
+        # This installCredentials script is written so that it's as easy as
+        # possible for a user to audit before confirming the `sudo`
+        installCredentials = hostPkgs.writeShellScript "install-credentials" ''
+          KEYS="''${1}"
+          INSTALL=${hostPkgs.coreutils}/bin/install
+          "''${INSTALL}" -g nixbld -m 600 "''${KEYS}/${user}_${keyType}" ${privateKey}
+          "''${INSTALL}" -g nixbld -m 644 "''${KEYS}/${user}_${keyType}.pub" ${publicKey}
+        '';
 
-      hostPkgs = config.virtualisation.host.pkgs;
+        hostPkgs = config.virtualisation.host.pkgs;
 
-      script = hostPkgs.writeShellScriptBin "create-builder" ''
-        KEYS="''${KEYS:-./keys}"
-        ${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
-        PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
-        PUBLIC_KEY="''${PRIVATE_KEY}.pub"
-        if [ ! -e "''${PRIVATE_KEY}" ] || [ ! -e "''${PUBLIC_KEY}" ]; then
-            ${hostPkgs.coreutils}/bin/rm --force -- "''${PRIVATE_KEY}" "''${PUBLIC_KEY}"
-            ${hostPkgs.openssh}/bin/ssh-keygen -q -f "''${PRIVATE_KEY}" -t ${keyType} -N "" -C 'builder@localhost'
-        fi
-        if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
-          (set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
-        fi
-        KEYS="$(nix-store --add "$KEYS")" ${config.system.build.vm}/bin/run-nixos-vm
-      '';
+  script = hostPkgs.writeShellScriptBin "create-builder" (
+          # When running as non-interactively as part of a DarwinConfiguration the working directory
+          # must be set to a writeable directory.
+        (if cfg.workingDirectory != "." then ''
+          ${hostPkgs.coreutils}/bin/mkdir --parent "${cfg.workingDirectory}"
+          cd "${cfg.workingDirectory}"
+  '' else "") + ''
+          KEYS="''${KEYS:-./keys}"
+          ${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
+          PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
+          PUBLIC_KEY="''${PRIVATE_KEY}.pub"
+          if [ ! -e "''${PRIVATE_KEY}" ] || [ ! -e "''${PUBLIC_KEY}" ]; then
+              ${hostPkgs.coreutils}/bin/rm --force -- "''${PRIVATE_KEY}" "''${PUBLIC_KEY}"
+              ${hostPkgs.openssh}/bin/ssh-keygen -q -f "''${PRIVATE_KEY}" -t ${keyType} -N "" -C 'builder@localhost'
+          fi
+          if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
+            (set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
+          fi
+          KEYS="$(${hostPkgs.nix}/bin/nix-store --add "$KEYS")" ${config.system.build.vm}/bin/run-nixos-vm
+        '');
 
-    in
-    script.overrideAttrs (old: {
-      meta = (old.meta or { }) // {
-        platforms = lib.platforms.darwin;
-      };
-    });
+      in
+      script.overrideAttrs (old: {
+        meta = (old.meta or { }) // {
+          platforms = lib.platforms.darwin;
+        };
+      });
 
-  system = {
-    # To prevent gratuitous rebuilds on each change to Nixpkgs
-    nixos.revision = null;
+    system = {
+      # To prevent gratuitous rebuilds on each change to Nixpkgs
+      nixos.revision = null;
 
-    stateVersion = lib.mkDefault (throw ''
-      The macOS linux builder should not need a stateVersion to be set, but a module
-      has accessed stateVersion nonetheless.
-      Please inspect the trace of the following command to figure out which module
-      has a dependency on stateVersion.
+      stateVersion = lib.mkDefault (throw ''
+        The macOS linux builder should not need a stateVersion to be set, but a module
+        has accessed stateVersion nonetheless.
+        Please inspect the trace of the following command to figure out which module
+        has a dependency on stateVersion.
 
-        nix-instantiate --attr darwin.builder --show-trace
-    '');
-  };
+          nix-instantiate --attr darwin.builder --show-trace
+      '');
+    };
 
-  users.users."${user}" = {
-    isNormalUser = true;
-  };
+    users.users."${user}" = {
+      isNormalUser = true;
+    };
 
-  security.polkit.enable = true;
+    security.polkit.enable = true;
 
-  security.polkit.extraConfig = ''
-    polkit.addRule(function(action, subject) {
-      if (action.id === "org.freedesktop.login1.power-off" && subject.user === "${user}") {
-        return "yes";
-      } else {
-        return "no";
-      }
-    })
-  '';
+    security.polkit.extraConfig = ''
+      polkit.addRule(function(action, subject) {
+        if (action.id === "org.freedesktop.login1.power-off" && subject.user === "${user}") {
+          return "yes";
+        } else {
+          return "no";
+        }
+      })
+    '';
 
-  virtualisation = {
-    diskSize = 20 * 1024;
+    virtualisation = {
+      diskSize = cfg.diskSize;
 
-    memorySize = 3 * 1024;
+      memorySize = cfg.memorySize;
 
-    forwardPorts = [
-      { from = "host"; guest.port = 22; host.port = 22; }
-    ];
+      forwardPorts = [
+        { from = "host"; guest.port = 22; host.port = cfg.hostPort; }
+      ];
 
-    # Disable graphics for the builder since users will likely want to run it
-    # non-interactively in the background.
-    graphics = false;
+      # Disable graphics for the builder since users will likely want to run it
+      # non-interactively in the background.
+      graphics = false;
 
-    sharedDirectories.keys = {
-      source = "\"$KEYS\"";
-      target = keysDirectory;
-    };
+      sharedDirectories.keys = {
+        source = "\"$KEYS\"";
+        target = keysDirectory;
+      };
 
-    # If we don't enable this option then the host will fail to delegate builds
-    # to the guest, because:
-    #
-    # - The host will lock the path to build
-    # - The host will delegate the build to the guest
-    # - The guest will attempt to lock the same path and fail because
-    #   the lockfile on the host is visible on the guest
-    #
-    # Snapshotting the host's /nix/store as an image isolates the guest VM's
-    # /nix/store from the host's /nix/store, preventing this problem.
-    useNixStoreImage = true;
+      # If we don't enable this option then the host will fail to delegate builds
+      # to the guest, because:
+      #
+      # - The host will lock the path to build
+      # - The host will delegate the build to the guest
+      # - The guest will attempt to lock the same path and fail because
+      #   the lockfile on the host is visible on the guest
+      #
+      # Snapshotting the host's /nix/store as an image isolates the guest VM's
+      # /nix/store from the host's /nix/store, preventing this problem.
+      useNixStoreImage = true;
 
-    # Obviously the /nix/store needs to be writable on the guest in order for it
-    # to perform builds.
-    writableStore = true;
+      # Obviously the /nix/store needs to be writable on the guest in order for it
+      # to perform builds.
+      writableStore = true;
 
-    # This ensures that anything built on the guest isn't lost when the guest is
-    # restarted.
-    writableStoreUseTmpfs = false;
+      # This ensures that anything built on the guest isn't lost when the guest is
+      # restarted.
+      writableStoreUseTmpfs = false;
+    };
   };
 }