diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/flake.lock b/flake.lock index e4c83c5..6ae2311 100644 --- a/flake.lock +++ b/flake.lock @@ -30,11 +30,11 @@ ] }, "locked": { - "lastModified": 1735048446, - "narHash": "sha256-Tc35Y8H+krA6rZeOIczsaGAtobSSBPqR32AfNTeHDRc=", + "lastModified": 1736165297, + "narHash": "sha256-OT+sF4eNDFN/OdyUfIQwyp28+CFQL7PAdWn0wGU7F0U=", "owner": "nix-community", "repo": "disko", - "rev": "3a4de9fa3a78ba7b7170dda6bd8b4cdab87c0b21", + "rev": "76816af65d5294761636a838917e335992a52e0c", "type": "github" }, "original": { @@ -45,11 +45,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1735444284, - "narHash": "sha256-U0Vw+ZrjbfvmHqeyJKM7lXZWUXIYdaOa32VtNKkfKo8=", + "lastModified": 1736165148, + "narHash": "sha256-AdKOlljgcTLOrJb3HFpaaoHWJhFrkVeT9HbRm0JvcwE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cf802984d6b41ea45044455724d5835a4f5fcf81", + "rev": "9f46f57b78d2ef865cd8c58eff8d430bb62a471a", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index c6e0123..d865005 100644 --- a/flake.nix +++ b/flake.nix @@ -23,22 +23,39 @@ inputs@{ self, nixpkgs, ... }: let kclib = import ./lib.nix { - nixpkgs = inputs.nixpkgs; inputs = inputs; }; in { nixosConfigurations = { - tailscale-proxy = kclib.mkHost { - name = "tailscale-proxy"; - }; - entrypoint = kclib.mkHost { - name = "entrypoint"; + bootstrap = kclib.mkHost { + name = "bootstrap"; }; hydra = kclib.mkHost { name = "hydra"; }; + node0 = kclib.mkHost { + name = "node0"; + }; + node1 = kclib.mkHost { + name = "node1"; + }; + node2 = kclib.mkHost { + name = "node2"; + }; }; formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-rfc-style; + devShells.x86_64-linux.default = + let + pkgs = import nixpkgs { + system = "x86_64-linux"; + allowUnfree = true; + }; + in + pkgs.mkShellNoCC { + packages = with pkgs; [ + cfssl + ]; + }; }; } diff --git a/hosts/base/hardware-config.nix b/hosts/base/hardware-config.nix index 781026e..8c4d9f2 100644 --- a/hosts/base/hardware-config.nix +++ b/hosts/base/hardware-config.nix @@ -1,7 +1,5 @@ { - config, lib, - pkgs, modulesPath, ... }: diff --git a/hosts/bootstrap/default.nix b/hosts/bootstrap/default.nix new file mode 100644 index 0000000..482a431 --- /dev/null +++ b/hosts/bootstrap/default.nix @@ -0,0 +1,10 @@ +{ ... }: +{ + kropcloud = { + networking.enable = false; + admin = { + password = "changeme"; + sudoRequirePassword = false; + }; + }; +} diff --git a/hosts/etcd0/default.nix b/hosts/etcd0/default.nix new file mode 100644 index 0000000..5ef2949 --- /dev/null +++ b/hosts/etcd0/default.nix @@ -0,0 +1,18 @@ +{ ... }: +{ + kropcloud = + let + serverIp = "192.168.1.161"; + in + { + services = { + }; + networking = { + ipv4 = { + address = serverIp; + prefixLength = 24; + defaultGateway = "192.168.1.1"; + }; + }; + }; +} diff --git a/hosts/node0/default.nix b/hosts/node0/default.nix new file mode 100644 index 0000000..aa4c27c --- /dev/null +++ b/hosts/node0/default.nix @@ -0,0 +1,22 @@ +{ ... }: +{ + kropcloud = + let + serverIp = "192.168.1.170"; + in + { + services = { + k3s = { + enable = true; + isMaster = true; + }; + }; + networking = { + ipv4 = { + address = serverIp; + prefixLength = 24; + defaultGateway = "192.168.1.1"; + }; + }; + }; +} diff --git a/hosts/node1/default.nix b/hosts/node1/default.nix new file mode 100644 index 0000000..6d37b63 --- /dev/null +++ b/hosts/node1/default.nix @@ -0,0 +1,22 @@ +{ ... }: +{ + kropcloud = + let + serverIp = "192.168.1.171"; + in + { + services = { + k3s = { + enable = true; + master = "node0"; + }; + }; + networking = { + ipv4 = { + address = serverIp; + prefixLength = 24; + defaultGateway = "192.168.1.1"; + }; + }; + }; +} diff --git a/hosts/node2/default.nix b/hosts/node2/default.nix new file mode 100644 index 0000000..f4119f1 --- /dev/null +++ b/hosts/node2/default.nix @@ -0,0 +1,22 @@ +{ ... }: +{ + kropcloud = + let + serverIp = "192.168.1.172"; + in + { + services = { + k3s = { + enable = true; + master = "node0"; + }; + }; + networking = { + ipv4 = { + address = serverIp; + prefixLength = 24; + defaultGateway = "192.168.1.1"; + }; + }; + }; +} diff --git a/lib.nix b/lib.nix index 95c26d5..3a31c0a 100644 --- a/lib.nix +++ b/lib.nix @@ -1,5 +1,4 @@ { - nixpkgs, inputs, }: { @@ -7,12 +6,13 @@ { name, arch ? "x86_64-linux", + config_name ? name, }: - nixpkgs.lib.nixosSystem { + inputs.nixpkgs.lib.nixosSystem { system = arch; modules = [ ./hosts/base - ./hosts/${name} + ./hosts/${config_name} ./nixosModules ( { ... }: @@ -29,4 +29,14 @@ inherit inputs; }; }; + # TODO: this will actually be nice, so I can see IPs in main flake.nix, + # but also dont have three directories with only default.nix in it + # mkK3Snode = { + # name_prefix, + # id, + # ip + # }: mkHost { + # name = "${name_prefix}-${id}"; + # config_name = "k3snode"; + # }; } diff --git a/nixosModules/networking/default.nix b/nixosModules/networking/default.nix index aa1878c..d52c704 100644 --- a/nixosModules/networking/default.nix +++ b/nixosModules/networking/default.nix @@ -49,6 +49,10 @@ in } ]; + services.avahi = { + enable = true; + }; + networking = { nftables.enable = true; firewall = { diff --git a/nixosModules/services/default.nix b/nixosModules/services/default.nix index edad497..5a9d92d 100644 --- a/nixosModules/services/default.nix +++ b/nixosModules/services/default.nix @@ -4,5 +4,6 @@ ./ssh ./tailscale ./hydra + ./k3s ]; } diff --git a/nixosModules/services/k3s/default.nix b/nixosModules/services/k3s/default.nix new file mode 100644 index 0000000..258f16b --- /dev/null +++ b/nixosModules/services/k3s/default.nix @@ -0,0 +1,58 @@ +{ + config, + lib, + ... +}: +let + cfg = config.kropcloud.services.k3s; +in +{ + options.kropcloud.services.k3s = { + enable = lib.mkEnableOption "Whence to enable k3s service."; + isMaster = lib.mkEnableOption "Whence to configure k3s as master."; + master = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + example = "node0"; + description = "The master node to connect to"; + }; + }; + config = lib.mkIf cfg.enable { + + assertions = [ + { + assertion = (!cfg.isMaster && cfg.master == null); + message = '' + You need to provide a valid value for `master` in `kropcloud.services.k3s` + when `isMaster` is not set. + ''; + } + ]; + + age.secrets.k3stoken.file = ../../../secrets/k3stoken.age; + + services.k3s = { + enable = true; + role = "server"; + tokenFile = config.age.secrets.k3stoken.path; + extraFlags = toString ( + [ + "--write-kubeconfig-mode \"0644\"" + "--cluster-init" + "--disable servicelb" + "--disable traefik" + "--disable local-storage" + ] + ++ ( + if cfg.isMaster && cfg.master != null then + [ ] + else + [ + "--server https://${cfg.master}:6443" + ] + ) + ); + clusterInit = cfg.isMaster; + }; + }; +} diff --git a/nixosModules/users/default.nix b/nixosModules/users/default.nix index 9add441..af557f8 100644 --- a/nixosModules/users/default.nix +++ b/nixosModules/users/default.nix @@ -14,18 +14,34 @@ in default = [ ]; description = "List of SSH public keys to authorize for the admin user."; }; + password = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Password for the admin user. Should be used only for initial setup."; + }; + sudoRequirePassword = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Require password for sudo. Should be used only for initial setup."; + }; }; config = { age.secrets.mypassword.file = ../../secrets/mypassword.age; + security.sudo.wheelNeedsPassword = cfg.sudoRequirePassword; + # Define the admin user - users.users.${cfg.user} = { - passwordFile = config.age.secrets.mypassword.path; - isNormalUser = true; - extraGroups = [ "wheel" ]; - openssh.authorizedKeys.keys = cfg.sshKeys; + users = { + mutableUsers = false; + users.${cfg.user} = { + password = if cfg.password != null then cfg.password else null; + hashedPasswordFile = if cfg.password != null then null else config.age.secrets.mypassword.path; + isNormalUser = true; + extraGroups = [ "wheel" ]; + openssh.authorizedKeys.keys = cfg.sshKeys; + }; }; }; } diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..07707bc --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Parameters +ip=$1 +if [ -z "$ip" ]; then + echo "Usage: $0 " + exit 1 +fi + +# This script is used to bootstrap nixos machine so I can get their ssh keys + +nix run \ + github:nix-community/nixos-anywhere \ + -- \ + --flake '.#bootstrap' \ + --target-host root@$ip \ + --build-on-remote + +ret=$? +if [ $ret -ne 0 ]; then + echo "Failed to bootstrap $ip" + exit $ret +fi \ No newline at end of file diff --git a/scripts/fresh_install.py b/scripts/fresh_install.py new file mode 100755 index 0000000..d49b419 --- /dev/null +++ b/scripts/fresh_install.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +import argparse +import json +import pathlib +import shutil +import socket +import time + +from subprocess import check_output + +ROOT_DIR = pathlib.Path(__file__).parent.parent +KEYS_FILE = ROOT_DIR / "secrets" / "keys.json" + + +def _get_available_machines() -> list: + output = check_output(["nix", "flake", "show", "--json"]) + parsed_output = json.loads(output) + machines = parsed_output.get("nixosConfigurations", dict()).keys() + return list(machines) + + +def _is_valid_ip(ip: str) -> bool: + try: + socket.inet_aton(ip) + return True + except socket.error: + return False + + +def _check_ssh_connection(ip: str) -> bool: + try: + check_output(["ssh", f"krop@{ip}", "echo", "Connected"]) + return True + except Exception: + return False + + +def add_key_to_secrets(machine_name: str, key: str): + keys = json.loads(KEYS_FILE.read_text()) + if keys.get("servers").get(machine_name): + raise ValueError(f"Key for {machine_name} already exists, remove it first") + + keys["servers"][machine_name] = key + + for secret in keys.get("secrets"): + keys["secrets"][secret].append(f"servers:{machine_name}") + + KEYS_FILE.write_text(json.dumps(keys, indent=2)) + + +def rekey_secrets(): + agenix_bin = shutil.which("agenix") + check_output([agenix_bin, "-r"], cwd=ROOT_DIR / "secrets") + + +def bootstrap_machine(ip: str): + check_output( + [ + "nix", + "run", + "github:nix-community/nixos-anywhere", + "--", + "--flake", + ".#bootstrap", + "--target-host", + f"root@{ip}", + "--build-on-remote", + ] + ) + + +def install_machine(machine_name: str, ip: str): + check_output( + [ + "nixos-rebuild", + "boot", + "--flake", + f".#{machine_name}", + "--fast", + "--target-host", + f"krop@{ip}", + "--build-host", + f"krop@{ip}", + "--use-remote-sudo", + ] + ) + + +def get_ssh_key(ip: str) -> str: + """ + This function uses machines ssh-keyscan to get the ssh key and then get the ed25519 key + """ + ssh_keys = ( + check_output( + [ + "ssh-keyscan", + "-q", + "-t", + "ed25519", + ip, + ] + ) + .decode("utf-8") + .strip() + .splitlines() + ) + + if len(ssh_keys) != 1: + raise ValueError("Exactly one key should be returned") + + key = ssh_keys.pop().lstrip(f"{ip} ").strip() + + return key + + +def get_machine_config(machine_name: str) -> dict: + output = check_output( + [ + "nix", + "eval", + "--json", + f".#nixosConfigurations.{machine_name}.config.kropcloud", + ] + ) + return json.loads(output) + + +def reboot_machine(ip: str): + check_output( + [ + "ssh", + f"krop@{ip}", + "sudo", + "reboot", + ] + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Install a machine") + parser.add_argument( + "machine_name", type=str, help="The name of the machine to install" + ) + parser.add_argument("machine_ip", type=str, help="The ip of the machine to install") + args = parser.parse_args() + + machine_name = args.machine_name + if machine_name not in _get_available_machines(): + raise ValueError( + f"Machine {machine_name} not found, available machines are: {_get_available_machines()}" + ) + + machine_ip = args.machine_ip + if not _is_valid_ip(machine_ip): + raise ValueError(f"Invalid IP address {machine_ip}") + + print(f"Bootstrapping machine {machine_ip}") + bootstrap_machine(machine_ip) + print("Machine bootstrapped") + + print("Waiting for ssh connection") + while not _check_ssh_connection(machine_ip): + time.sleep(5) + print("Machine is up and running") + + print("Getting ssh key") + ssh_key = get_ssh_key(machine_ip) + + print(f"SSH key: {ssh_key}") + print("Adding ssh key to secrets") + add_key_to_secrets(machine_name, ssh_key) + rekey_secrets() + + print("Installing machine") + install_machine(machine_name, machine_ip) + print("Machine installed, rebooting") + reboot_machine(machine_ip) + print("") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..9c5ad04 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Parameters +ip=$1 +host=$2 +if [ -z "$ip" ] || [ -z "$host" ]; then + echo "Usage: $0 " + exit 1 +fi + +# This script is used to bootstrap nixos machine so I can get their ssh keys + +nixos-rebuild boot \ + --flake ".#$host" \ + --fast \ + --target-host krop@$ip \ + --build-host krop@$ip \ + --use-remote-sudo + +ret=$? +if [ $ret -ne 0 ]; then + echo "Failed to install $host" + exit $ret +fi + +echo "Successfully installed $host, rebooting" + +ssh -t krop@$ip "sudo reboot now" + +ret=$? +if [ $ret -ne 0 ]; then + echo "Failed to reboot $host" + exit $ret +fi \ No newline at end of file diff --git a/scripts/update.sh b/scripts/update.sh new file mode 100755 index 0000000..c12b64b --- /dev/null +++ b/scripts/update.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Parameters +ip=$1 +if [ -z "$ip" ]; then + echo "Usage: $0 " + exit 1 +fi + +# This script is used to bootstrap nixos machine so I can get their ssh keys + +nixos-rebuild switch \ + --flake ".#$host" \ + --fast \ + --target-host krop@$ip \ + --build-host krop@$ip \ + --use-remote-sudo + +ret=$? +if [ $ret -ne 0 ]; then + echo "Failed to update $ip" + exit $ret +fi + +echo "Successfully updated $ip, rebooting" \ No newline at end of file diff --git a/secrets/k3stoken.age b/secrets/k3stoken.age new file mode 100644 index 0000000..c7da7fc --- /dev/null +++ b/secrets/k3stoken.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> ssh-ed25519 5k28aQ wUKJk8gcxcCqbdXsfuod3dvEtj+pXRe8rLYVv/uyND4 +aHOXSUwP5+AJZ5etU+dj9ssVNQNcDuXSpq+wvIYsoyE +-> ssh-ed25519 MhDGlw Ln5f8TTQFDlp+KGQpRRPNgn/+fzoY7Bnl7FlDg5ZSSs +uJbxZFjjcSxhIPHvregG1tD8BKKfHHMlvfZ6itDIppY +--- MGApTU7O6xSlpanV9LC22ZX2u7bwULpBMaTLg01SO/0 +Y J#ž6/ 6 wTF fԶ xם5^ \ No newline at end of file diff --git a/secrets/keys.json b/secrets/keys.json new file mode 100644 index 0000000..2f613a1 --- /dev/null +++ b/secrets/keys.json @@ -0,0 +1,20 @@ +{ + "hosts": { + "wenar-nix": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJl0Rdo2kHliBeIiPuiO4kYO5M0VZFNXw4siepV1p6Pj", + "lenar": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOUnlAjPnMwJYgZb7YuholdTxifOEFnAyXVqI+xFlHw6" + }, + "servers": { + "test-server": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID4ioqiTzYe6Y6H0YfFkWyDBbCB25wYs3gKNZIufE/Sn" + }, + "secrets": { + "mypassword.age": [ + "hosts:wenar-nix", + "hosts:lenar", + "servers:test-server" + ], + "k3stoken.age": [ + "hosts:wenar-nix", + "hosts:lenar" + ] + } +} diff --git a/secrets/mypassword.age b/secrets/mypassword.age index 6c6df84..6ae2b05 100644 Binary files a/secrets/mypassword.age and b/secrets/mypassword.age differ diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 069a97b..fdb4274 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -1,17 +1,13 @@ let - wenar-nix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJl0Rdo2kHliBeIiPuiO4kYO5M0VZFNXw4siepV1p6Pj"; - lenar = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOUnlAjPnMwJYgZb7YuholdTxifOEFnAyXVqI+xFlHw6"; - users = [ - wenar-nix - lenar - ]; + keyfile = builtins.fromJSON (builtins.readFile ./keys.json); - test-server = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID4ioqiTzYe6Y6H0YfFkWyDBbCB25wYs3gKNZIufE/Sn"; - systems = [ - test-server - ]; - allKeys = users ++ systems; + splitString = delim: str: builtins.filter builtins.isString (builtins.split delim str); + + getKey = pair: keyfile.${builtins.elemAt pair 0}.${builtins.elemAt pair 1}; + + getKeys = secretName: builtins.map (x: getKey (splitString ":" x)) keyfile.secrets.${secretName}; in { - "mypassword.age".publicKeys = allKeys; + "mypassword.age".publicKeys = getKeys "mypassword.age"; + "k3stoken.age".publicKeys = getKeys "k3stoken.age"; }