Merge pull request 'Add ability to bootstrap new machines without the need of manual intervention' (#1) from bootstrapping into master

Reviewed-on: #1

.envrc

@@ -0,0 +1 @@
+use flake

flake.lock

@@ -30,11 +30,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1735048446,
-        "narHash": "sha256-Tc35Y8H+krA6rZeOIczsaGAtobSSBPqR32AfNTeHDRc=",
+        "lastModified": 1736165297,
+        "narHash": "sha256-OT+sF4eNDFN/OdyUfIQwyp28+CFQL7PAdWn0wGU7F0U=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "3a4de9fa3a78ba7b7170dda6bd8b4cdab87c0b21",
+        "rev": "76816af65d5294761636a838917e335992a52e0c",
         "type": "github"
       },
       "original": {
@@ -45,11 +45,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1735444284,
-        "narHash": "sha256-U0Vw+ZrjbfvmHqeyJKM7lXZWUXIYdaOa32VtNKkfKo8=",
+        "lastModified": 1736165148,
+        "narHash": "sha256-AdKOlljgcTLOrJb3HFpaaoHWJhFrkVeT9HbRm0JvcwE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cf802984d6b41ea45044455724d5835a4f5fcf81",
+        "rev": "9f46f57b78d2ef865cd8c58eff8d430bb62a471a",
         "type": "github"
       },
       "original": {

flake.nix

@@ -23,22 +23,39 @@
     inputs@{ self, nixpkgs, ... }:
     let
       kclib = import ./lib.nix {
-        nixpkgs = inputs.nixpkgs;
         inputs = inputs;
       };
     in
     {
       nixosConfigurations = {
-        tailscale-proxy = kclib.mkHost {
-          name = "tailscale-proxy";
-        };
-        entrypoint = kclib.mkHost {
-          name = "entrypoint";
+        bootstrap = kclib.mkHost {
+          name = "bootstrap";
         };
         hydra = kclib.mkHost {
           name = "hydra";
         };
+        node0 = kclib.mkHost {
+          name = "node0";
+        };
+        node1 = kclib.mkHost {
+          name = "node1";
+        };
+        node2 = kclib.mkHost {
+          name = "node2";
+        };
       };
       formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-rfc-style;
+      devShells.x86_64-linux.default =
+        let
+          pkgs = import nixpkgs {
+            system = "x86_64-linux";
+            allowUnfree = true;
+          };
+        in
+        pkgs.mkShellNoCC {
+          packages = with pkgs; [
+            cfssl
+          ];
+        };
     };
 }

hosts/base/hardware-config.nix

@@ -1,7 +1,5 @@
 {
-  config,
   lib,
-  pkgs,
   modulesPath,
   ...
 }:

hosts/bootstrap/default.nix

@@ -0,0 +1,10 @@
+{ ... }:
+{
+  kropcloud = {
+    networking.enable = false;
+    admin = {
+      password = "changeme";
+      sudoRequirePassword = false;
+    };
+  };
+}

hosts/etcd0/default.nix

@@ -0,0 +1,18 @@
+{ ... }:
+{
+  kropcloud =
+    let
+      serverIp = "192.168.1.161";
+    in
+    {
+      services = {
+      };
+      networking = {
+        ipv4 = {
+          address = serverIp;
+          prefixLength = 24;
+          defaultGateway = "192.168.1.1";
+        };
+      };
+    };
+}

hosts/node0/default.nix

@@ -0,0 +1,22 @@
+{ ... }:
+{
+  kropcloud =
+    let
+      serverIp = "192.168.1.170";
+    in
+    {
+      services = {
+        k3s = {
+          enable = true;
+          isMaster = true;
+        };
+      };
+      networking = {
+        ipv4 = {
+          address = serverIp;
+          prefixLength = 24;
+          defaultGateway = "192.168.1.1";
+        };
+      };
+    };
+}

hosts/node1/default.nix

@@ -0,0 +1,22 @@
+{ ... }:
+{
+  kropcloud =
+    let
+      serverIp = "192.168.1.171";
+    in
+    {
+      services = {
+        k3s = {
+          enable = true;
+          master = "node0";
+        };
+      };
+      networking = {
+        ipv4 = {
+          address = serverIp;
+          prefixLength = 24;
+          defaultGateway = "192.168.1.1";
+        };
+      };
+    };
+}

hosts/node2/default.nix

@@ -0,0 +1,22 @@
+{ ... }:
+{
+  kropcloud =
+    let
+      serverIp = "192.168.1.172";
+    in
+    {
+      services = {
+        k3s = {
+          enable = true;
+          master = "node0";
+        };
+      };
+      networking = {
+        ipv4 = {
+          address = serverIp;
+          prefixLength = 24;
+          defaultGateway = "192.168.1.1";
+        };
+      };
+    };
+}

lib.nix

@@ -1,5 +1,4 @@
 {
-  nixpkgs,
   inputs,
 }:
 {
@@ -7,12 +6,13 @@
     {
       name,
       arch ? "x86_64-linux",
+      config_name ? name,
     }:
-    nixpkgs.lib.nixosSystem {
+    inputs.nixpkgs.lib.nixosSystem {
       system = arch;
       modules = [
         ./hosts/base
-        ./hosts/${name}
+        ./hosts/${config_name}
         ./nixosModules
         (
           { ... }:
@@ -29,4 +29,14 @@
         inherit inputs;
       };
     };
+  # TODO: this will actually be nice, so I can see IPs in main flake.nix,
+  # but also dont have three directories with only default.nix in it
+  # mkK3Snode = {
+  #   name_prefix,
+  #   id,
+  #   ip
+  # }: mkHost {
+  #   name = "${name_prefix}-${id}";
+  #   config_name = "k3snode";
+  # };
 }

nixosModules/networking/default.nix

@@ -49,6 +49,10 @@ in
       }
     ];
 
+    services.avahi = {
+      enable = true;
+    };
+
     networking = {
       nftables.enable = true;
       firewall = {

nixosModules/services/default.nix

@@ -4,5 +4,6 @@
     ./ssh
     ./tailscale
     ./hydra
+    ./k3s
   ];
 }

nixosModules/services/k3s/default.nix

@@ -0,0 +1,58 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.kropcloud.services.k3s;
+in
+{
+  options.kropcloud.services.k3s = {
+    enable = lib.mkEnableOption "Whence to enable k3s service.";
+    isMaster = lib.mkEnableOption "Whence to configure k3s as master.";
+    master = lib.mkOption {
+      type = with lib.types; nullOr str;
+      default = null;
+      example = "node0";
+      description = "The master node to connect to";
+    };
+  };
+  config = lib.mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion = (!cfg.isMaster && cfg.master == null);
+        message = ''
+          You need to provide a valid value for `master` in `kropcloud.services.k3s`
+          when `isMaster` is not set.
+        '';
+      }
+    ];
+
+    age.secrets.k3stoken.file = ../../../secrets/k3stoken.age;
+
+    services.k3s = {
+      enable = true;
+      role = "server";
+      tokenFile = config.age.secrets.k3stoken.path;
+      extraFlags = toString (
+        [
+          "--write-kubeconfig-mode \"0644\""
+          "--cluster-init"
+          "--disable servicelb"
+          "--disable traefik"
+          "--disable local-storage"
+        ]
+        ++ (
+          if cfg.isMaster && cfg.master != null then
+            [ ]
+          else
+            [
+              "--server https://${cfg.master}:6443"
+            ]
+        )
+      );
+      clusterInit = cfg.isMaster;
+    };
+  };
+}

nixosModules/users/default.nix

@@ -14,18 +14,34 @@ in
       default = [ ];
       description = "List of SSH public keys to authorize for the admin user.";
     };
+    password = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = "Password for the admin user. Should be used only for initial setup.";
+    };
+    sudoRequirePassword = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Require password for sudo. Should be used only for initial setup.";
+    };
   };
 
   config = {
 
     age.secrets.mypassword.file = ../../secrets/mypassword.age;
 
+    security.sudo.wheelNeedsPassword = cfg.sudoRequirePassword;
+
     # Define the admin user
-    users.users.${cfg.user} = {
-      passwordFile = config.age.secrets.mypassword.path;
-      isNormalUser = true;
-      extraGroups = [ "wheel" ];
-      openssh.authorizedKeys.keys = cfg.sshKeys;
+    users = {
+      mutableUsers = false;
+      users.${cfg.user} = {
+        password = if cfg.password != null then cfg.password else null;
+        hashedPasswordFile = if cfg.password != null then null else config.age.secrets.mypassword.path;
+        isNormalUser = true;
+        extraGroups = [ "wheel" ];
+        openssh.authorizedKeys.keys = cfg.sshKeys;
+      };
     };
   };
 }

scripts/bootstrap.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Parameters
+ip=$1
+if [ -z "$ip" ]; then
+    echo "Usage: $0 <ip>"
+    exit 1
+fi
+
+# This script is used to bootstrap nixos machine so I can get their ssh keys
+
+nix run \
+    github:nix-community/nixos-anywhere \
+    -- \
+    --flake '.#bootstrap' \
+    --target-host root@$ip \
+    --build-on-remote
+
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "Failed to bootstrap $ip"
+    exit $ret
+fi

scripts/fresh_install.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import pathlib
+import shutil
+import socket
+import time
+
+from subprocess import check_output
+
+ROOT_DIR = pathlib.Path(__file__).parent.parent
+KEYS_FILE = ROOT_DIR / "secrets" / "keys.json"
+
+
+def _get_available_machines() -> list:
+    output = check_output(["nix", "flake", "show", "--json"])
+    parsed_output = json.loads(output)
+    machines = parsed_output.get("nixosConfigurations", dict()).keys()
+    return list(machines)
+
+
+def _is_valid_ip(ip: str) -> bool:
+    try:
+        socket.inet_aton(ip)
+        return True
+    except socket.error:
+        return False
+
+
+def _check_ssh_connection(ip: str) -> bool:
+    try:
+        check_output(["ssh", f"krop@{ip}", "echo", "Connected"])
+        return True
+    except Exception:
+        return False
+
+
+def add_key_to_secrets(machine_name: str, key: str):
+    keys = json.loads(KEYS_FILE.read_text())
+    if keys.get("servers").get(machine_name):
+        raise ValueError(f"Key for {machine_name} already exists, remove it first")
+
+    keys["servers"][machine_name] = key
+
+    for secret in keys.get("secrets"):
+        keys["secrets"][secret].append(f"servers:{machine_name}")
+
+    KEYS_FILE.write_text(json.dumps(keys, indent=2))
+
+
+def rekey_secrets():
+    agenix_bin = shutil.which("agenix")
+    check_output([agenix_bin, "-r"], cwd=ROOT_DIR / "secrets")
+
+
+def bootstrap_machine(ip: str):
+    check_output(
+        [
+            "nix",
+            "run",
+            "github:nix-community/nixos-anywhere",
+            "--",
+            "--flake",
+            ".#bootstrap",
+            "--target-host",
+            f"root@{ip}",
+            "--build-on-remote",
+        ]
+    )
+
+
+def install_machine(machine_name: str, ip: str):
+    check_output(
+        [
+            "nixos-rebuild",
+            "boot",
+            "--flake",
+            f".#{machine_name}",
+            "--fast",
+            "--target-host",
+            f"krop@{ip}",
+            "--build-host",
+            f"krop@{ip}",
+            "--use-remote-sudo",
+        ]
+    )
+
+
+def get_ssh_key(ip: str) -> str:
+    """
+    This function uses machines ssh-keyscan to get the ssh key and then get the ed25519 key
+    """
+    ssh_keys = (
+        check_output(
+            [
+                "ssh-keyscan",
+                "-q",
+                "-t",
+                "ed25519",
+                ip,
+            ]
+        )
+        .decode("utf-8")
+        .strip()
+        .splitlines()
+    )
+
+    if len(ssh_keys) != 1:
+        raise ValueError("Exactly one key should be returned")
+
+    key = ssh_keys.pop().lstrip(f"{ip} ").strip()
+
+    return key
+
+
+def get_machine_config(machine_name: str) -> dict:
+    output = check_output(
+        [
+            "nix",
+            "eval",
+            "--json",
+            f".#nixosConfigurations.{machine_name}.config.kropcloud",
+        ]
+    )
+    return json.loads(output)
+
+
+def reboot_machine(ip: str):
+    check_output(
+        [
+            "ssh",
+            f"krop@{ip}",
+            "sudo",
+            "reboot",
+        ]
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Install a machine")
+    parser.add_argument(
+        "machine_name", type=str, help="The name of the machine to install"
+    )
+    parser.add_argument("machine_ip", type=str, help="The ip of the machine to install")
+    args = parser.parse_args()
+
+    machine_name = args.machine_name
+    if machine_name not in _get_available_machines():
+        raise ValueError(
+            f"Machine {machine_name} not found, available machines are: {_get_available_machines()}"
+        )
+
+    machine_ip = args.machine_ip
+    if not _is_valid_ip(machine_ip):
+        raise ValueError(f"Invalid IP address {machine_ip}")
+
+    print(f"Bootstrapping machine {machine_ip}")
+    bootstrap_machine(machine_ip)
+    print("Machine bootstrapped")
+
+    print("Waiting for ssh connection")
+    while not _check_ssh_connection(machine_ip):
+        time.sleep(5)
+    print("Machine is up and running")
+
+    print("Getting ssh key")
+    ssh_key = get_ssh_key(machine_ip)
+
+    print(f"SSH key: {ssh_key}")
+    print("Adding ssh key to secrets")
+    add_key_to_secrets(machine_name, ssh_key)
+    rekey_secrets()
+
+    print("Installing machine")
+    install_machine(machine_name, machine_ip)
+    print("Machine installed, rebooting")
+    reboot_machine(machine_ip)
+    print("")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/install.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Parameters
+ip=$1
+host=$2
+if [ -z "$ip" ] || [ -z "$host" ]; then
+    echo "Usage: $0 <ip> <host>"
+    exit 1
+fi
+
+# This script is used to bootstrap nixos machine so I can get their ssh keys
+
+nixos-rebuild boot \
+    --flake ".#$host" \
+    --fast \
+    --target-host krop@$ip \
+    --build-host krop@$ip \
+    --use-remote-sudo
+
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "Failed to install $host"
+    exit $ret
+fi
+
+echo "Successfully installed $host, rebooting"
+
+ssh -t krop@$ip "sudo reboot now"
+
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "Failed to reboot $host"
+    exit $ret
+fi

scripts/update.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Parameters
+ip=$1
+if [ -z "$ip" ]; then
+    echo "Usage: $0 <ip>"
+    exit 1
+fi
+
+# This script is used to bootstrap nixos machine so I can get their ssh keys
+
+nixos-rebuild switch \
+    --flake ".#$host" \
+    --fast \
+    --target-host krop@$ip \
+    --build-host krop@$ip \
+    --use-remote-sudo
+
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "Failed to update $ip"
+    exit $ret
+fi
+
+echo "Successfully updated $ip, rebooting"

secrets/k3stoken.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 5k28aQ wUKJk8gcxcCqbdXsfuod3dvEtj+pXRe8rLYVv/uyND4
+aHOXSUwP5+AJZ5etU+dj9ssVNQNcDuXSpq+wvIYsoyE
+-> ssh-ed25519 MhDGlw Ln5f8TTQFDlp+KGQpRRPNgn/+fzoY7Bnl7FlDg5ZSSs
+uJbxZFjjcSxhIPHvregG1tD8BKKfHHMlvfZ6itDIppY
+--- MGApTU7O6xSlpanV9LC22ZX2u7bwULpBMaTLg01SO/0
+šâYøï	ö¯J#<23>ž6/ó—6 ñwTF¯ìfŒÔ¶¡	x×<78>º™5·Îÿ¸^

secrets/keys.json

@@ -0,0 +1,20 @@
+{
+  "hosts": {
+    "wenar-nix": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJl0Rdo2kHliBeIiPuiO4kYO5M0VZFNXw4siepV1p6Pj",
+    "lenar": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOUnlAjPnMwJYgZb7YuholdTxifOEFnAyXVqI+xFlHw6"
+  },
+  "servers": {
+    "test-server": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID4ioqiTzYe6Y6H0YfFkWyDBbCB25wYs3gKNZIufE/Sn"
+  },
+  "secrets": {
+    "mypassword.age": [
+      "hosts:wenar-nix",
+      "hosts:lenar",
+      "servers:test-server"
+    ],
+    "k3stoken.age": [
+      "hosts:wenar-nix",
+      "hosts:lenar"
+    ]
+  }
+}

secrets/secrets.nix

@@ -1,17 +1,13 @@
 let
-  wenar-nix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJl0Rdo2kHliBeIiPuiO4kYO5M0VZFNXw4siepV1p6Pj";
-  lenar = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOUnlAjPnMwJYgZb7YuholdTxifOEFnAyXVqI+xFlHw6";
-  users = [
-    wenar-nix
-    lenar
-  ];
+  keyfile = builtins.fromJSON (builtins.readFile ./keys.json);
 
-  test-server = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID4ioqiTzYe6Y6H0YfFkWyDBbCB25wYs3gKNZIufE/Sn";
-  systems = [
-    test-server
-  ];
-  allKeys = users ++ systems;
+  splitString = delim: str: builtins.filter builtins.isString (builtins.split delim str);
+
+  getKey = pair: keyfile.${builtins.elemAt pair 0}.${builtins.elemAt pair 1};
+
+  getKeys = secretName: builtins.map (x: getKey (splitString ":" x)) keyfile.secrets.${secretName};
 in
 {
-  "mypassword.age".publicKeys = allKeys;
+  "mypassword.age".publicKeys = getKeys "mypassword.age";
+  "k3stoken.age".publicKeys = getKeys "k3stoken.age";
 }