From d30e84aad5baf6ecc4f76e4d6db6967544a77a79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 29 Aug 2022 14:11:38 +0200
Subject: [PATCH] switch to deploykit from deploy_nixos
---
deploy_nixos.py | 253 ------------------------------------------------
flake.lock | 45 +++++++++
flake.nix | 133 +++++++++++++++----------
shell.nix | 4 +-
tasks.py | 2 +-
5 files changed, 129 insertions(+), 308 deletions(-)
delete mode 100644 deploy_nixos.py
diff --git a/deploy_nixos.py b/deploy_nixos.py
deleted file mode 100644
index 7733f9d..0000000
--- a/deploy_nixos.py
+++ /dev/null
@@ -1,253 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-from contextlib import contextmanager
-from typing import List, Dict, Tuple, IO, Iterator, Optional, Callable, Any, Union, Text
-from threading import Thread
-import subprocess
-import fcntl
-import select
-from contextlib import ExitStack
-from enum import Enum
-
-
-@contextmanager
-def pipe() -> Iterator[Tuple[IO[str], IO[str]]]:
- (pipe_r, pipe_w) = os.pipe()
- read_end = os.fdopen(pipe_r, "r")
- write_end = os.fdopen(pipe_w, "w")
-
- try:
- fl = fcntl.fcntl(read_end, fcntl.F_GETFL)
- fcntl.fcntl(read_end, fcntl.F_SETFL, fl | os.O_NONBLOCK)
-
- yield (read_end, write_end)
- finally:
- read_end.close()
- write_end.close()
-
-
-FILE = Union[None, int]
-
-
-class HostKeyCheck(Enum):
- STRICT = 0
- # trust-on-first-use
- TOFU = 1
- NONE = 2
-
-
-class DeployHost:
- def __init__(
- self,
- host: str,
- user: str = "root",
- port: int = 22,
- forward_agent: bool = False,
- command_prefix: Optional[str] = None,
- host_key_check: HostKeyCheck = HostKeyCheck.STRICT,
- meta: Dict[str, Any] = {},
- ) -> None:
- self.host = host
- self.user = user
- self.port = port
- if command_prefix:
- self.command_prefix = command_prefix
- else:
- self.command_prefix = host
- self.forward_agent = forward_agent
- self.host_key_check = host_key_check
- self.meta = meta
-
- def _prefix_output(
- self, print_fd: IO[str], stdout: Optional[IO[str]], stderr: Optional[IO[str]]
- ) -> Tuple[str, str]:
- rlist = [print_fd]
- if stdout is not None:
- rlist.append(stdout)
-
- if stderr is not None:
- rlist.append(stderr)
-
- print_buf = ""
- stdout_buf = ""
- stderr_buf = ""
-
- while len(rlist) != 0:
- r, _, _ = select.select(rlist, [], [])
-
- if print_fd in r:
- read = os.read(print_fd.fileno(), 4096)
- if len(read) == 0:
- rlist.remove(print_fd)
- print_buf += read.decode("utf-8")
- if read == "" or "\n" in print_buf:
- lines = print_buf.rstrip("\n").split("\n")
- for line in lines:
- print(f"[{self.command_prefix}] {line}")
- print_buf = ""
-
- def handle_fd(fd: Optional[IO[Any]]) -> str:
- if fd and fd in r:
- read = os.read(fd.fileno(), 4096)
- if len(read) == 0:
- rlist.remove(fd)
- else:
- return read.decode("utf-8")
- return ""
-
- stdout_buf += handle_fd(stdout)
- stderr_buf += handle_fd(stderr)
- return stdout_buf, stderr_buf
-
- def _run(
- self, cmd: List[str], shell: bool, stdout: FILE = None, stderr: FILE = None
- ) -> subprocess.CompletedProcess[Text]:
- with ExitStack() as stack:
- if stdout is None or stderr is None:
- read_fd, write_fd = stack.enter_context(pipe())
-
- if stdout is None:
- stdout_read = None
- stdout_write = write_fd
- elif stdout == subprocess.PIPE:
- stdout_read, stdout_write = stack.enter_context(pipe())
-
- if stderr is None:
- stderr_read = None
- stderr_write = write_fd
- elif stderr == subprocess.PIPE:
- stderr_read, stderr_write = stack.enter_context(pipe())
-
- with subprocess.Popen(
- cmd, text=True, shell=shell, stdout=stdout_write, stderr=stderr_write
- ) as p:
- write_fd.close()
- if stdout == subprocess.PIPE:
- stdout_write.close()
- if stderr == subprocess.PIPE:
- stderr_write.close()
- stdout_data, stderr_data = self._prefix_output(read_fd, stdout_read, stderr_read)
- ret = p.wait()
- return subprocess.CompletedProcess(
- cmd, ret, stdout=stdout_data, stderr=stderr_data
- )
- raise RuntimeError("unreachable")
-
- def run_local(
- self, cmd: str, stdout: FILE = None, stderr: FILE = None
- ) -> subprocess.CompletedProcess:
- print(f"[{self.command_prefix}] {cmd}")
- return self._run([cmd], shell=True, stdout=stdout, stderr=stderr)
-
- def run(
- self, cmd: str, stdout: FILE = None, stderr: FILE = None
- ) -> subprocess.CompletedProcess:
- print(f"[{self.command_prefix}] {cmd}")
- ssh_opts = ["-A"] if self.forward_agent else []
-
- if self.host_key_check != HostKeyCheck.STRICT:
- ssh_opts.extend(["-o", "StrictHostKeyChecking=no"])
- if self.host_key_check == HostKeyCheck.NONE:
- ssh_opts.extend(["-o", "UserKnownHostsFile=/dev/null"])
-
- ssh_cmd = (
- ["ssh", f"{self.user}@{self.host}", "-p", str(self.port)]
- + ssh_opts
- + ["--", cmd]
- )
- return self._run(ssh_cmd, shell=False, stdout=stdout, stderr=stderr)
-
-
-DeployResults = List[Tuple[DeployHost, subprocess.CompletedProcess[Text]]]
-
-
-class DeployGroup:
- def __init__(self, hosts: List[DeployHost]) -> None:
- self.hosts = hosts
-
- def _run_local(
- self,
- cmd: str,
- host: DeployHost,
- results: DeployResults,
- stdout: FILE = None,
- stderr: FILE = None,
- ) -> None:
- results.append((host, host.run_local(cmd, stdout=stdout, stderr=stderr)))
-
- def _run_remote(
- self,
- cmd: str,
- host: DeployHost,
- results: DeployResults,
- stdout: FILE = None,
- stderr: FILE = None,
- ) -> None:
- results.append((host, host.run(cmd, stdout=stdout, stderr=stderr)))
-
- def _run(
- self, cmd: str, local: bool = False, stdout: FILE = None, stderr: FILE = None
- ) -> DeployResults:
- results: DeployResults = []
- threads = []
- for host in self.hosts:
- fn = self._run_local if local else self._run_remote
- thread = Thread(
- target=fn,
- kwargs=dict(
- results=results, cmd=cmd, host=host, stdout=stdout, stderr=stderr
- ),
- )
- thread.start()
- threads.append(thread)
-
- for thread in threads:
- thread.join()
-
- return results
-
- def run(self, cmd: str, stdout: FILE = None, stderr: FILE = None) -> DeployResults:
- return self._run(cmd, stdout=stdout, stderr=stderr)
-
- def run_local(
- self, cmd: str, stdout: FILE = None, stderr: FILE = None
- ) -> DeployResults:
- return self._run(cmd, local=True, stdout=stdout, stderr=stderr)
-
- def run_function(self, func: Callable) -> None:
- threads = []
- for host in self.hosts:
- thread = Thread(
- target=func,
- args=(host,),
- )
- threads.append(thread)
-
- for thread in threads:
- thread.start()
-
- for thread in threads:
- thread.join()
-
-
-def parse_hosts(
- hosts: str,
- host_key_check: HostKeyCheck = HostKeyCheck.STRICT,
- forward_agent: bool = False,
-) -> DeployGroup:
- deploy_hosts = []
- for h in hosts.split(","):
- parts = h.split("@")
- if len(parts) > 1:
- user = parts[0]
- hostname = parts[1]
- else:
- user = "root"
- hostname = parts[0]
- deploy_hosts.append(
- DeployHost(
- hostname, user=user, host_key_check=host_key_check, forward_agent=forward_agent
- )
- )
- return DeployGroup(deploy_hosts)
diff --git a/flake.lock b/flake.lock
index e58dec7..901daf9 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,5 +1,28 @@
{
"nodes": {
+ "deploykit": {
+ "inputs": {
+ "flake-parts": [
+ "flake-parts"
+ ],
+ "nixpkgs": [
+ "nixpkgs"
+ ]
+ },
+ "locked": {
+ "lastModified": 1661774097,
+ "narHash": "sha256-dvWpOU3dU9TfSnK7I+CSDrCaASr+uR5VepG/SQs/RIg=",
+ "owner": "numtide",
+ "repo": "deploykit",
+ "rev": "c8cd9ddcd647e3116b07e920e53aebf047cbeac5",
+ "type": "github"
+ },
+ "original": {
+ "owner": "numtide",
+ "repo": "deploykit",
+ "type": "github"
+ }
+ },
"flake-compat": {
"flake": false,
"locked": {
@@ -16,6 +39,26 @@
"type": "github"
}
},
+ "flake-parts": {
+ "inputs": {
+ "nixpkgs": [
+ "nixpkgs"
+ ]
+ },
+ "locked": {
+ "lastModified": 1661009076,
+ "narHash": "sha256-phAE40gctVygRq3G3B6LhvD7u2qdQT21xsz8DdRDYFo=",
+ "owner": "hercules-ci",
+ "repo": "flake-parts",
+ "rev": "850d8a76026127ef02f040fb0dcfdb8b749dd9d9",
+ "type": "github"
+ },
+ "original": {
+ "owner": "hercules-ci",
+ "repo": "flake-parts",
+ "type": "github"
+ }
+ },
"hercules-ci-effects": {
"inputs": {
"nixpkgs": "nixpkgs"
@@ -310,6 +353,8 @@
},
"root": {
"inputs": {
+ "deploykit": "deploykit",
+ "flake-parts": "flake-parts",
"hercules-ci-effects": "hercules-ci-effects",
"hydra": "hydra",
"nixpkgs": "nixpkgs_3",
diff --git a/flake.nix b/flake.nix
index 320ff6d..8bb5c06 100644
--- a/flake.nix
+++ b/flake.nix
@@ -21,63 +21,90 @@
hercules-ci-effects.url = "github:hercules-ci/hercules-ci-effects";
hydra.url = "github:NixOS/hydra";
hydra.inputs.nixpkgs.follows = "nixpkgs";
+
+ flake-parts.url = "github:hercules-ci/flake-parts";
+ flake-parts.inputs.nixpkgs.follows = "nixpkgs";
+
+ deploykit.url = "github:numtide/deploykit";
+ deploykit.inputs.nixpkgs.follows = "nixpkgs";
+ deploykit.inputs.flake-parts.follows = "flake-parts";
};
- outputs = { self
- , nixpkgs
- , nixpkgs-update
- , nixpkgs-update-github-releases
- , nixpkgs-update-pypi-releases
- , sops-nix
- , hercules-ci-effects
- , hydra
- }: {
- devShell.x86_64-linux = let
- pkgs = nixpkgs.legacyPackages.x86_64-linux;
- in pkgs.callPackage ./shell.nix {
- inherit (sops-nix.packages.x86_64-linux) sops-import-keys-hook;
- };
- nixosConfigurations = let
- common = [
- sops-nix.nixosModules.sops
- ];
- in {
- "build01.nix-community.org" = nixpkgs.lib.nixosSystem {
- system = "x86_64-linux";
- modules = common ++ [
- ./build01/configuration.nix
- ];
- };
+ outputs = {
+ self,
+ flake-parts,
+ ...
+ }:
+ (flake-parts.lib.evalFlakeModule
+ {inherit self;}
+ {
+ systems = ["x86_64-linux" "aarch64-linux" "x86_64-linux" "aarch64-darwin"];
- "build02.nix-community.org" = nixpkgs.lib.nixosSystem {
- system = "x86_64-linux";
- modules = common ++ [
- (import ./build02/nixpkgs-update.nix {
- inherit nixpkgs-update
- nixpkgs-update-github-releases
- nixpkgs-update-pypi-releases;
- })
- ./build02/configuration.nix
- ];
- };
+ perSystem = {
+ inputs',
+ pkgs,
+ ...
+ }: {
+ devShells.default = pkgs.callPackage ./shell.nix {
+ inherit (inputs'.sops-nix.packages) sops-import-keys-hook;
+ inherit (inputs'.deploykit.packages) deploykit;
+ };
+ };
+ flake.nixosConfigurations = let
+ inherit (self.inputs.nixpkgs.lib) nixosSystem;
+ common = [
+ self.inputs.sops-nix.nixosModules.sops
+ ];
+ in {
+ "build01.nix-community.org" = nixosSystem {
+ system = "x86_64-linux";
+ modules =
+ common
+ ++ [
+ ./build01/configuration.nix
+ ];
+ };
- "build03.nix-community.org" = nixpkgs.lib.nixosSystem {
- system = "x86_64-linux";
- modules = common ++ [
- (import ./services/hydra {
- inherit hydra;
- })
+ "build02.nix-community.org" = nixosSystem {
+ system = "x86_64-linux";
+ modules =
+ common
+ ++ [
+ (import ./build02/nixpkgs-update.nix {
+ inherit
+ (self.inputs)
+ nixpkgs-update
+ nixpkgs-update-github-releases
+ nixpkgs-update-pypi-releases
+ ;
+ })
+ ./build02/configuration.nix
+ ];
+ };
- ./build03/configuration.nix
- ];
- };
+ "build03.nix-community.org" = nixosSystem {
+ system = "x86_64-linux";
+ modules =
+ common
+ ++ [
+ (import ./services/hydra {
+ inherit (self.inputs) hydra;
+ })
- "build04.nix-community.org" = nixpkgs.lib.nixosSystem {
- system = "aarch64-linux";
- modules = common ++ [
- ./build04/configuration.nix
- ];
- };
- };
- };
+ ./build03/configuration.nix
+ ];
+ };
+
+ "build04.nix-community.org" = nixosSystem {
+ system = "aarch64-linux";
+ modules =
+ common
+ ++ [
+ ./build04/configuration.nix
+ ];
+ };
+ };
+ })
+ .config
+ .flake;
}
diff --git a/shell.nix b/shell.nix
index 19935d0..aafdcf2 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,9 +1,10 @@
{ pkgs ? import <nixpkgs> {}
, sops-import-keys-hook
+, deploykit
}:
with pkgs;
-mkShell {
+mkShellNoCC {
sopsPGPKeyDirs = [
"./keys"
];
@@ -23,5 +24,6 @@ mkShell {
rsync
sops-import-keys-hook
+ deploykit
];
}
diff --git a/tasks.py b/tasks.py
index dc14a6a..ff64382 100644
--- a/tasks.py
+++ b/tasks.py
@@ -4,7 +4,7 @@ from invoke import task
import sys
from typing import List, Any
-from deploy_nixos import DeployHost, DeployGroup
+from deploykit import DeployHost, DeployGroup
import subprocess
import json