infra/hosts/build02/nixpkgs-update.nix
2024-07-24 10:27:26 +00:00

317 lines
10 KiB
Nix

{
pkgs,
lib,
inputs,
config,
...
}:
let
userLib = import "${toString inputs.self}/users/lib.nix" { inherit lib; };
nixpkgs-update-bin = "/var/lib/nixpkgs-update/bin/nixpkgs-update";
nixpkgsUpdateSystemDependencies = with pkgs; [
nix # for nix-shell used by python packges to update fetchers
git # used by update-scripts
openssh # used by git
gnugrep
gnused
curl
getent # used by hub
cachix
apacheHttpd # for rotatelogs, used by worker script
socat # used by worker script
];
mkWorker = name: {
after = [
"network-online.target"
"nixpkgs-update-supervisor.service"
];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
description = "nixpkgs-update ${name} service";
enable = true;
restartIfChanged = true;
path = nixpkgsUpdateSystemDependencies;
environment.XDG_CONFIG_HOME = "/var/lib/nixpkgs-update/worker";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/worker";
environment.XDG_RUNTIME_DIR = "/run/nixpkgs-update-worker"; # for nix-update update scripts
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
Restart = "on-failure";
RestartSec = "5s";
WorkingDirectory = "/var/lib/nixpkgs-update/worker";
StateDirectory = "nixpkgs-update/worker";
StateDirectoryMode = "700";
CacheDirectory = "nixpkgs-update/worker";
CacheDirectoryMode = "700";
LogsDirectory = "nixpkgs-update/";
LogsDirectoryMode = "755";
RuntimeDirectory = "nixpkgs-update-worker";
RuntimeDirectoryMode = "700";
StandardOutput = "journal";
};
script = ''
mkdir -p "$LOGS_DIRECTORY/~workers/"
# This is for public logs at nixpkgs-update-logs.nix-community.org/~workers
exec > >(rotatelogs -eD "$LOGS_DIRECTORY"'/~workers/%Y-%m-%d-${name}.stdout.log' 86400)
exec 2> >(rotatelogs -eD "$LOGS_DIRECTORY"'/~workers/%Y-%m-%d-${name}.stderr.log' 86400 >&2)
socket=/run/nixpkgs-update-supervisor/work.sock
function run-nixpkgs-update {
exit_code=0
set -x
timeout 6h ${nixpkgs-update-bin} update-batch --pr --outpaths --nixpkgs-review "$attr_path $payload" || exit_code=$?
set +x
if [ $exit_code -eq 124 ]; then
echo "Update was interrupted because it was taking too long."
fi
msg="DONE $attr_path $exit_code"
}
msg=READY
while true; do
response=$(echo "$msg" | socat -t5 UNIX-CONNECT:"$socket" - || true)
case "$response" in
"") # connection error; retry
sleep 5
;;
NOJOBS)
msg=READY
sleep 60
;;
JOB\ *)
read -r attr_path payload <<< "''${response#JOB }"
# If one worker is initializing the nixpkgs clone, the other will
# try to use the incomplete clone, consuming a bunch of jobs and
# throwing them away. So we use a crude locking mechanism to
# run only one worker when there isn't a nixpkgs directory yet.
# Once the directory exists and this initial lock is released,
# multiple workers can run concurrently.
lockdir="$XDG_CACHE_HOME/.nixpkgs.lock"
if [ ! -e "$XDG_CACHE_HOME/nixpkgs" ] && mkdir "$lockdir"; then
trap 'rmdir "$lockdir"' EXIT
run-nixpkgs-update
rmdir "$lockdir"
trap - EXIT
continue
fi
while [ -e "$lockdir" ]; do
sleep 10
done
run-nixpkgs-update
esac
done
'';
};
mkFetcher = name: cmd: {
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = nixpkgsUpdateSystemDependencies ++ [
# nixpkgs-update-github-releases
(pkgs.python3.withPackages (
p: with p; [
requests
dateutil
libversion
cachecontrol
lockfile
filelock
]
))
];
# API_TOKEN is used by nixpkgs-update-github-releases
# using a token from another account so the rate limit doesn't block opening PRs
environment.API_TOKEN_FILE = "${config.sops.secrets.github-token-with-username.path}";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/fetcher/";
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
Restart = "on-failure";
RestartSec = "30m";
LogsDirectory = "nixpkgs-update/";
LogsDirectoryMode = "755";
StateDirectory = "nixpkgs-update";
StateDirectoryMode = "700";
CacheDirectory = "nixpkgs-update/fetcher";
CacheDirectoryMode = "700";
};
script = ''
mkdir -p "$LOGS_DIRECTORY/~fetchers"
cd "$LOGS_DIRECTORY/~fetchers"
run_name="${name}.$(date +%s).txt"
rm -f ${name}.*.txt.part
${cmd} > "$run_name.part"
rm -f ${name}.*.txt
mv "$run_name.part" "$run_name"
'';
startAt = "0/12:10"; # every 12 hours
};
in
{
users.groups.r-ryantm = { };
users.users.r-ryantm = {
useDefaultShell = true;
isNormalUser = true; # The hub cli seems to really want stuff to be set up like a normal user
uid = userLib.mkUid "rrtm";
extraGroups = [ "r-ryantm" ];
};
systemd.services.nixpkgs-update-delete-done = {
startAt = "0/12:10"; # every 12 hours
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
description = "nixpkgs-update delete done branches";
restartIfChanged = true;
path = nixpkgsUpdateSystemDependencies;
environment.XDG_CONFIG_HOME = "/var/lib/nixpkgs-update/worker";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/worker";
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
Restart = "on-abort";
RestartSec = "5s";
WorkingDirectory = "/var/lib/nixpkgs-update/worker";
StateDirectory = "nixpkgs-update/worker";
StateDirectoryMode = "700";
CacheDirectoryMode = "700";
LogsDirectory = "nixpkgs-update/";
LogsDirectoryMode = "755";
StandardOutput = "journal";
};
script = "${nixpkgs-update-bin} delete-done --delete";
};
systemd.services.nixpkgs-update-fetch-repology = mkFetcher "repology" "${nixpkgs-update-bin} fetch-repology";
systemd.services.nixpkgs-update-fetch-updatescript = mkFetcher "updatescript" "${pkgs.nix}/bin/nix eval --raw -f ${./packages-with-update-script.nix}";
systemd.services.nixpkgs-update-fetch-github = mkFetcher "github" "${inputs.nixpkgs-update-github-releases}/main.py";
systemd.services.nixpkgs-update-worker1 = mkWorker "worker1";
systemd.services.nixpkgs-update-worker2 = mkWorker "worker2";
systemd.services.nixpkgs-update-worker3 = mkWorker "worker3";
systemd.services.nixpkgs-update-worker4 = mkWorker "worker4";
# Too many workers cause out-of-memory.
systemd.services.nixpkgs-update-supervisor = {
wantedBy = [ "multi-user.target" ];
description = "nixpkgs-update supervisor service";
enable = true;
restartIfChanged = true;
path = with pkgs; [
apacheHttpd
(python3.withPackages (ps: [ ps.asyncinotify ]))
];
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
Restart = "on-failure";
RestartSec = "5s";
LogsDirectory = "nixpkgs-update/";
LogsDirectoryMode = "755";
RuntimeDirectory = "nixpkgs-update-supervisor/";
RuntimeDirectoryMode = "755";
StandardOutput = "journal";
};
script = ''
mkdir -p "$LOGS_DIRECTORY/~supervisor"
# This is for public logs at nixpkgs-update-logs.nix-community.org/~supervisor
exec > >(rotatelogs -eD "$LOGS_DIRECTORY"'/~supervisor/%Y-%m-%d.stdout.log' 86400)
exec 2> >(rotatelogs -eD "$LOGS_DIRECTORY"'/~supervisor/%Y-%m-%d.stderr.log' 86400 >&2)
# Fetcher output is hosted at nixpkgs-update-logs.nix-community.org/~fetchers
python3 ${./supervisor.py} "$LOGS_DIRECTORY/~supervisor/state.db" "$LOGS_DIRECTORY/~fetchers" "$RUNTIME_DIRECTORY/work.sock"
'';
};
systemd.services.nixpkgs-update-delete-old-logs = {
startAt = "daily";
# delete logs older than 18 months, delete worker logs older than 3 months, delete empty directories
script = ''
${pkgs.findutils}/bin/find /var/log/nixpkgs-update -type f -mtime +548 -delete
${pkgs.findutils}/bin/find /var/log/nixpkgs-update/~workers -type f -mtime +90 -delete
${pkgs.findutils}/bin/find /var/log/nixpkgs-update -type d -empty -delete
'';
serviceConfig.Type = "oneshot";
};
systemd.tmpfiles.rules = [
"L+ /home/r-ryantm/.gitconfig - - - - ${./gitconfig.txt}"
"d /home/r-ryantm/.ssh 700 r-ryantm r-ryantm - -"
"e /var/cache/nixpkgs-update/worker/nixpkgs-review - - - 1d -"
"d /var/lib/nixpkgs-update/bin/ 700 r-ryantm r-ryantm - -"
"L+ ${nixpkgs-update-bin} - - - - ${
inputs.nixpkgs-update.packages.${pkgs.system}.default
}/bin/nixpkgs-update"
];
sops.secrets.github-r-ryantm-key = {
path = "/home/r-ryantm/.ssh/id_rsa";
owner = "r-ryantm";
group = "r-ryantm";
};
sops.secrets.github-r-ryantm-token = {
path = "/var/lib/nixpkgs-update/worker/github_token.txt";
owner = "r-ryantm";
group = "r-ryantm";
};
sops.secrets.github-token-with-username = {
owner = "r-ryantm";
group = "r-ryantm";
};
sops.secrets.nix-community-cachix = {
path = "/var/lib/nixpkgs-update/worker/cachix/cachix.dhall";
owner = "r-ryantm";
group = "r-ryantm";
};
# autoindex is truncated on some browsers
services.nginx.recommendedZstdSettings = false;
services.nginx.virtualHosts."nixpkgs-update-logs.nix-community.org" = {
forceSSL = true;
enableACME = true;
locations."/" = {
alias = "/var/log/nixpkgs-update/";
extraConfig = ''
charset utf-8;
autoindex on;
'';
};
};
# TODO: permanent redirect r.ryantm.com/log/ -> nixpkgs-update-logs.nix-community.org
services.nginx.virtualHosts."r.ryantm.com" = {
forceSSL = true;
enableACME = true;
locations."/log/" = {
alias = "/var/log/nixpkgs-update/";
extraConfig = ''
charset utf-8;
autoindex on;
'';
};
};
}