switch to deploykit from deploy_nixos
This commit is contained in:
Jörg Thalheim
parent
2387d97c5b
commit
d30e84aad5
5 changed files with 129 additions and 308 deletions
253
deploy_nixos.py
253
deploy_nixos.py
|
|
@ -1,253 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from typing import List, Dict, Tuple, IO, Iterator, Optional, Callable, Any, Union, Text
|
||||
from threading import Thread
|
||||
import subprocess
|
||||
import fcntl
|
||||
import select
|
||||
from contextlib import ExitStack
|
||||
from enum import Enum
|
||||
|
||||
|
||||
@contextmanager
|
||||
def pipe() -> Iterator[Tuple[IO[str], IO[str]]]:
|
||||
(pipe_r, pipe_w) = os.pipe()
|
||||
read_end = os.fdopen(pipe_r, "r")
|
||||
write_end = os.fdopen(pipe_w, "w")
|
||||
|
||||
try:
|
||||
fl = fcntl.fcntl(read_end, fcntl.F_GETFL)
|
||||
fcntl.fcntl(read_end, fcntl.F_SETFL, fl | os.O_NONBLOCK)
|
||||
|
||||
yield (read_end, write_end)
|
||||
finally:
|
||||
read_end.close()
|
||||
write_end.close()
|
||||
|
||||
|
||||
FILE = Union[None, int]
|
||||
|
||||
|
||||
class HostKeyCheck(Enum):
|
||||
STRICT = 0
|
||||
# trust-on-first-use
|
||||
TOFU = 1
|
||||
NONE = 2
|
||||
|
||||
|
||||
class DeployHost:
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
user: str = "root",
|
||||
port: int = 22,
|
||||
forward_agent: bool = False,
|
||||
command_prefix: Optional[str] = None,
|
||||
host_key_check: HostKeyCheck = HostKeyCheck.STRICT,
|
||||
meta: Dict[str, Any] = {},
|
||||
) -> None:
|
||||
self.host = host
|
||||
self.user = user
|
||||
self.port = port
|
||||
if command_prefix:
|
||||
self.command_prefix = command_prefix
|
||||
else:
|
||||
self.command_prefix = host
|
||||
self.forward_agent = forward_agent
|
||||
self.host_key_check = host_key_check
|
||||
self.meta = meta
|
||||
|
||||
def _prefix_output(
|
||||
self, print_fd: IO[str], stdout: Optional[IO[str]], stderr: Optional[IO[str]]
|
||||
) -> Tuple[str, str]:
|
||||
rlist = [print_fd]
|
||||
if stdout is not None:
|
||||
rlist.append(stdout)
|
||||
|
||||
if stderr is not None:
|
||||
rlist.append(stderr)
|
||||
|
||||
print_buf = ""
|
||||
stdout_buf = ""
|
||||
stderr_buf = ""
|
||||
|
||||
while len(rlist) != 0:
|
||||
r, _, _ = select.select(rlist, [], [])
|
||||
|
||||
if print_fd in r:
|
||||
read = os.read(print_fd.fileno(), 4096)
|
||||
if len(read) == 0:
|
||||
rlist.remove(print_fd)
|
||||
print_buf += read.decode("utf-8")
|
||||
if read == "" or "\n" in print_buf:
|
||||
lines = print_buf.rstrip("\n").split("\n")
|
||||
for line in lines:
|
||||
print(f"[{self.command_prefix}] {line}")
|
||||
print_buf = ""
|
||||
|
||||
def handle_fd(fd: Optional[IO[Any]]) -> str:
|
||||
if fd and fd in r:
|
||||
read = os.read(fd.fileno(), 4096)
|
||||
if len(read) == 0:
|
||||
rlist.remove(fd)
|
||||
else:
|
||||
return read.decode("utf-8")
|
||||
return ""
|
||||
|
||||
stdout_buf += handle_fd(stdout)
|
||||
stderr_buf += handle_fd(stderr)
|
||||
return stdout_buf, stderr_buf
|
||||
|
||||
def _run(
|
||||
self, cmd: List[str], shell: bool, stdout: FILE = None, stderr: FILE = None
|
||||
) -> subprocess.CompletedProcess[Text]:
|
||||
with ExitStack() as stack:
|
||||
if stdout is None or stderr is None:
|
||||
read_fd, write_fd = stack.enter_context(pipe())
|
||||
|
||||
if stdout is None:
|
||||
stdout_read = None
|
||||
stdout_write = write_fd
|
||||
elif stdout == subprocess.PIPE:
|
||||
stdout_read, stdout_write = stack.enter_context(pipe())
|
||||
|
||||
if stderr is None:
|
||||
stderr_read = None
|
||||
stderr_write = write_fd
|
||||
elif stderr == subprocess.PIPE:
|
||||
stderr_read, stderr_write = stack.enter_context(pipe())
|
||||
|
||||
with subprocess.Popen(
|
||||
cmd, text=True, shell=shell, stdout=stdout_write, stderr=stderr_write
|
||||
) as p:
|
||||
write_fd.close()
|
||||
if stdout == subprocess.PIPE:
|
||||
stdout_write.close()
|
||||
if stderr == subprocess.PIPE:
|
||||
stderr_write.close()
|
||||
stdout_data, stderr_data = self._prefix_output(read_fd, stdout_read, stderr_read)
|
||||
ret = p.wait()
|
||||
return subprocess.CompletedProcess(
|
||||
cmd, ret, stdout=stdout_data, stderr=stderr_data
|
||||
)
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
def run_local(
|
||||
self, cmd: str, stdout: FILE = None, stderr: FILE = None
|
||||
) -> subprocess.CompletedProcess:
|
||||
print(f"[{self.command_prefix}] {cmd}")
|
||||
return self._run([cmd], shell=True, stdout=stdout, stderr=stderr)
|
||||
|
||||
def run(
|
||||
self, cmd: str, stdout: FILE = None, stderr: FILE = None
|
||||
) -> subprocess.CompletedProcess:
|
||||
print(f"[{self.command_prefix}] {cmd}")
|
||||
ssh_opts = ["-A"] if self.forward_agent else []
|
||||
|
||||
if self.host_key_check != HostKeyCheck.STRICT:
|
||||
ssh_opts.extend(["-o", "StrictHostKeyChecking=no"])
|
||||
if self.host_key_check == HostKeyCheck.NONE:
|
||||
ssh_opts.extend(["-o", "UserKnownHostsFile=/dev/null"])
|
||||
|
||||
ssh_cmd = (
|
||||
["ssh", f"{self.user}@{self.host}", "-p", str(self.port)]
|
||||
+ ssh_opts
|
||||
+ ["--", cmd]
|
||||
)
|
||||
return self._run(ssh_cmd, shell=False, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
DeployResults = List[Tuple[DeployHost, subprocess.CompletedProcess[Text]]]
|
||||
|
||||
|
||||
class DeployGroup:
|
||||
def __init__(self, hosts: List[DeployHost]) -> None:
|
||||
self.hosts = hosts
|
||||
|
||||
def _run_local(
|
||||
self,
|
||||
cmd: str,
|
||||
host: DeployHost,
|
||||
results: DeployResults,
|
||||
stdout: FILE = None,
|
||||
stderr: FILE = None,
|
||||
) -> None:
|
||||
results.append((host, host.run_local(cmd, stdout=stdout, stderr=stderr)))
|
||||
|
||||
def _run_remote(
|
||||
self,
|
||||
cmd: str,
|
||||
host: DeployHost,
|
||||
results: DeployResults,
|
||||
stdout: FILE = None,
|
||||
stderr: FILE = None,
|
||||
) -> None:
|
||||
results.append((host, host.run(cmd, stdout=stdout, stderr=stderr)))
|
||||
|
||||
def _run(
|
||||
self, cmd: str, local: bool = False, stdout: FILE = None, stderr: FILE = None
|
||||
) -> DeployResults:
|
||||
results: DeployResults = []
|
||||
threads = []
|
||||
for host in self.hosts:
|
||||
fn = self._run_local if local else self._run_remote
|
||||
thread = Thread(
|
||||
target=fn,
|
||||
kwargs=dict(
|
||||
results=results, cmd=cmd, host=host, stdout=stdout, stderr=stderr
|
||||
),
|
||||
)
|
||||
thread.start()
|
||||
threads.append(thread)
|
||||
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
return results
|
||||
|
||||
def run(self, cmd: str, stdout: FILE = None, stderr: FILE = None) -> DeployResults:
|
||||
return self._run(cmd, stdout=stdout, stderr=stderr)
|
||||
|
||||
def run_local(
|
||||
self, cmd: str, stdout: FILE = None, stderr: FILE = None
|
||||
) -> DeployResults:
|
||||
return self._run(cmd, local=True, stdout=stdout, stderr=stderr)
|
||||
|
||||
def run_function(self, func: Callable) -> None:
|
||||
threads = []
|
||||
for host in self.hosts:
|
||||
thread = Thread(
|
||||
target=func,
|
||||
args=(host,),
|
||||
)
|
||||
threads.append(thread)
|
||||
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
|
||||
def parse_hosts(
|
||||
hosts: str,
|
||||
host_key_check: HostKeyCheck = HostKeyCheck.STRICT,
|
||||
forward_agent: bool = False,
|
||||
) -> DeployGroup:
|
||||
deploy_hosts = []
|
||||
for h in hosts.split(","):
|
||||
parts = h.split("@")
|
||||
if len(parts) > 1:
|
||||
user = parts[0]
|
||||
hostname = parts[1]
|
||||
else:
|
||||
user = "root"
|
||||
hostname = parts[0]
|
||||
deploy_hosts.append(
|
||||
DeployHost(
|
||||
hostname, user=user, host_key_check=host_key_check, forward_agent=forward_agent
|
||||
)
|
||||
)
|
||||
return DeployGroup(deploy_hosts)
|
||||
45
flake.lock
generated
45
flake.lock
generated
|
|
@ -1,5 +1,28 @@
|
|||
{
|
||||
"nodes": {
|
||||
"deploykit": {
|
||||
"inputs": {
|
||||
"flake-parts": [
|
||||
"flake-parts"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1661774097,
|
||||
"narHash": "sha256-dvWpOU3dU9TfSnK7I+CSDrCaASr+uR5VepG/SQs/RIg=",
|
||||
"owner": "numtide",
|
||||
"repo": "deploykit",
|
||||
"rev": "c8cd9ddcd647e3116b07e920e53aebf047cbeac5",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "deploykit",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
|
|
@ -16,6 +39,26 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-parts": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1661009076,
|
||||
"narHash": "sha256-phAE40gctVygRq3G3B6LhvD7u2qdQT21xsz8DdRDYFo=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "850d8a76026127ef02f040fb0dcfdb8b749dd9d9",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"hercules-ci-effects": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
|
|
@ -310,6 +353,8 @@
|
|||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"deploykit": "deploykit",
|
||||
"flake-parts": "flake-parts",
|
||||
"hercules-ci-effects": "hercules-ci-effects",
|
||||
"hydra": "hydra",
|
||||
"nixpkgs": "nixpkgs_3",
|
||||
|
|
|
|||
133
flake.nix
133
flake.nix
|
|
@ -21,63 +21,90 @@
|
|||
hercules-ci-effects.url = "github:hercules-ci/hercules-ci-effects";
|
||||
hydra.url = "github:NixOS/hydra";
|
||||
hydra.inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
||||
flake-parts.url = "github:hercules-ci/flake-parts";
|
||||
flake-parts.inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
||||
deploykit.url = "github:numtide/deploykit";
|
||||
deploykit.inputs.nixpkgs.follows = "nixpkgs";
|
||||
deploykit.inputs.flake-parts.follows = "flake-parts";
|
||||
};
|
||||
|
||||
outputs = { self
|
||||
, nixpkgs
|
||||
, nixpkgs-update
|
||||
, nixpkgs-update-github-releases
|
||||
, nixpkgs-update-pypi-releases
|
||||
, sops-nix
|
||||
, hercules-ci-effects
|
||||
, hydra
|
||||
}: {
|
||||
devShell.x86_64-linux = let
|
||||
pkgs = nixpkgs.legacyPackages.x86_64-linux;
|
||||
in pkgs.callPackage ./shell.nix {
|
||||
inherit (sops-nix.packages.x86_64-linux) sops-import-keys-hook;
|
||||
};
|
||||
nixosConfigurations = let
|
||||
common = [
|
||||
sops-nix.nixosModules.sops
|
||||
];
|
||||
in {
|
||||
"build01.nix-community.org" = nixpkgs.lib.nixosSystem {
|
||||
system = "x86_64-linux";
|
||||
modules = common ++ [
|
||||
./build01/configuration.nix
|
||||
];
|
||||
};
|
||||
outputs = {
|
||||
self,
|
||||
flake-parts,
|
||||
...
|
||||
}:
|
||||
(flake-parts.lib.evalFlakeModule
|
||||
{inherit self;}
|
||||
{
|
||||
systems = ["x86_64-linux" "aarch64-linux" "x86_64-linux" "aarch64-darwin"];
|
||||
|
||||
"build02.nix-community.org" = nixpkgs.lib.nixosSystem {
|
||||
system = "x86_64-linux";
|
||||
modules = common ++ [
|
||||
(import ./build02/nixpkgs-update.nix {
|
||||
inherit nixpkgs-update
|
||||
nixpkgs-update-github-releases
|
||||
nixpkgs-update-pypi-releases;
|
||||
})
|
||||
./build02/configuration.nix
|
||||
];
|
||||
};
|
||||
perSystem = {
|
||||
inputs',
|
||||
pkgs,
|
||||
...
|
||||
}: {
|
||||
devShells.default = pkgs.callPackage ./shell.nix {
|
||||
inherit (inputs'.sops-nix.packages) sops-import-keys-hook;
|
||||
inherit (inputs'.deploykit.packages) deploykit;
|
||||
};
|
||||
};
|
||||
flake.nixosConfigurations = let
|
||||
inherit (self.inputs.nixpkgs.lib) nixosSystem;
|
||||
common = [
|
||||
self.inputs.sops-nix.nixosModules.sops
|
||||
];
|
||||
in {
|
||||
"build01.nix-community.org" = nixosSystem {
|
||||
system = "x86_64-linux";
|
||||
modules =
|
||||
common
|
||||
++ [
|
||||
./build01/configuration.nix
|
||||
];
|
||||
};
|
||||
|
||||
"build03.nix-community.org" = nixpkgs.lib.nixosSystem {
|
||||
system = "x86_64-linux";
|
||||
modules = common ++ [
|
||||
(import ./services/hydra {
|
||||
inherit hydra;
|
||||
})
|
||||
"build02.nix-community.org" = nixosSystem {
|
||||
system = "x86_64-linux";
|
||||
modules =
|
||||
common
|
||||
++ [
|
||||
(import ./build02/nixpkgs-update.nix {
|
||||
inherit
|
||||
(self.inputs)
|
||||
nixpkgs-update
|
||||
nixpkgs-update-github-releases
|
||||
nixpkgs-update-pypi-releases
|
||||
;
|
||||
})
|
||||
./build02/configuration.nix
|
||||
];
|
||||
};
|
||||
|
||||
./build03/configuration.nix
|
||||
];
|
||||
};
|
||||
"build03.nix-community.org" = nixosSystem {
|
||||
system = "x86_64-linux";
|
||||
modules =
|
||||
common
|
||||
++ [
|
||||
(import ./services/hydra {
|
||||
inherit (self.inputs) hydra;
|
||||
})
|
||||
|
||||
"build04.nix-community.org" = nixpkgs.lib.nixosSystem {
|
||||
system = "aarch64-linux";
|
||||
modules = common ++ [
|
||||
./build04/configuration.nix
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
./build03/configuration.nix
|
||||
];
|
||||
};
|
||||
|
||||
"build04.nix-community.org" = nixosSystem {
|
||||
system = "aarch64-linux";
|
||||
modules =
|
||||
common
|
||||
++ [
|
||||
./build04/configuration.nix
|
||||
];
|
||||
};
|
||||
};
|
||||
})
|
||||
.config
|
||||
.flake;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
{ pkgs ? import <nixpkgs> {}
|
||||
, sops-import-keys-hook
|
||||
, deploykit
|
||||
}:
|
||||
|
||||
with pkgs;
|
||||
mkShell {
|
||||
mkShellNoCC {
|
||||
sopsPGPKeyDirs = [
|
||||
"./keys"
|
||||
];
|
||||
|
|
@ -23,5 +24,6 @@ mkShell {
|
|||
rsync
|
||||
|
||||
sops-import-keys-hook
|
||||
deploykit
|
||||
];
|
||||
}
|
||||
|
|
|
|||
2
tasks.py
2
tasks.py
|
|
@ -4,7 +4,7 @@ from invoke import task
|
|||
|
||||
import sys
|
||||
from typing import List, Any
|
||||
from deploy_nixos import DeployHost, DeployGroup
|
||||
from deploykit import DeployHost, DeployGroup
|
||||
import subprocess
|
||||
import json
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue