sercanto/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

nixpkgs-update: use fetcher -> fifo -> worker model

Browse source
This commit is contained in:
Ryan Mulligan 2022-07-27 19:34:53 -07:00
parent 75a9db784e
commit 37b2fbef90
2 changed files with 112 additions and 64 deletions

170
build02/nixpkgs-update.nix
View file

@ -21,39 +21,87 @@ let
nixpkgs-update-github-releases' = "${nixpkgs-update-github-releases}/main.py"; nixpkgs-update-github-releases' = "${nixpkgs-update-github-releases}/main.py";
nixpkgs-update-pypi-releases' = "${nixpkgs-update-pypi-releases}/main.py"; nixpkgs-update-pypi-releases' = "${nixpkgs-update-pypi-releases}/main.py";
mkNixpkgsUpdateService = name: { mkWorker = name: {
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
restartTriggers = [ nixpkgs-update-bin ];
description = "nixpkgs-update ${name} service"; description = "nixpkgs-update ${name} service";
enable = true; enable = true;
startAt = "daily"; restartIfChanged = true;
restartIfChanged = false;
path = nixpkgsUpdateSystemDependencies; path = nixpkgsUpdateSystemDependencies;
environment.XDG_CONFIG_HOME = "/var/lib/nixpkgs-update/${name}"; environment.XDG_CONFIG_HOME = "/var/lib/nixpkgs-update/worker";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/${name}"; environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/worker";
environment.XDG_RUNTIME_DIR = "/run/nixpkgs-update/${name}"; # for nix-update update scripts environment.XDG_RUNTIME_DIR = "/run/nixpkgs-update/worker"; # for nix-update update scripts
# API_TOKEN is used by nixpkgs-update-github-releases
environment.API_TOKEN_FILE = "/var/lib/nixpkgs-update/github_token_with_username.txt";
# Used by nixpkgs-update-github-releases to install python dependencies
# Used by nixpkgs-update-pypi-releases
environment.NIX_PATH = "nixpkgs=/var/cache/nixpkgs-update/${name}/nixpkgs";
serviceConfig = { serviceConfig = {
Type = "simple"; Type = "simple";
User = "r-ryantm"; User = "r-ryantm";
Group = "r-ryantm"; Group = "r-ryantm";
WorkingDirectory = "/var/lib/nixpkgs-update/${name}"; Restart = "on-abort";
StateDirectory = "nixpkgs-update/${name}"; RestartSec = "5s";
WorkingDirectory = "/var/lib/nixpkgs-update/worker";
StateDirectory = "nixpkgs-update/worker";
StateDirectoryMode = "700"; StateDirectoryMode = "700";
CacheDirectory = "nixpkgs-update/${name}"; CacheDirectory = "nixpkgs-update/worker";
CacheDirectoryMode = "700"; CacheDirectoryMode = "700";
LogsDirectory = "nixpkgs-update/${name}"; LogsDirectory = "nixpkgs-update/";
LogsDirectoryMode = "755"; LogsDirectoryMode = "755";
RuntimeDirectory = "nixpkgs-update/${name}"; RuntimeDirectory = "nixpkgs-update/worker";
RuntimeDirectoryMode = "700"; RuntimeDirectoryMode = "700";
StandardOutput = "journal"; StandardOutput = "journal";
}; };
script = ''
pipe=/var/lib/nixpkgs-update/fifo
if [[ ! -p $pipe ]]; then
mkfifo $pipe || true
fi
exec 8<$pipe
while true
do
if read -u 8 line; then
set -x
${nixpkgs-update-bin} update-batch --pr --outpaths --nixpkgs-review "$line" || true
set +x
fi
done
'';
}; };
nixpkgs-update-command = "${nixpkgs-update-bin} update-list --pr --outpaths --nixpkgs-review"; mkFetcher = cmd: {
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
restartTriggers = [ nixpkgs-update-bin ];
path = nixpkgsUpdateSystemDependencies;
# API_TOKEN is used by nixpkgs-update-github-releases
environment.API_TOKEN_FILE = "/var/lib/nixpkgs-update/github_token_with_username.txt";
# Used by nixpkgs-update-github-releases to install python dependencies
# Used by nixpkgs-update-pypi-releases
environment.NIX_PATH = "nixpkgs=/var/cache/nixpkgs-update/fetcher/nixpkgs";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/fetcher/";
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
Restart = "on-abort";
RestartSec = "5s";
WorkingDirectory = "/var/lib/nixpkgs-update/";
StateDirectory = "nixpkgs-update";
StateDirectoryMode = "700";
};
script = ''
pipe=/var/lib/nixpkgs-update/fifo
if [[ ! -p $pipe ]]; then
mkfifo $pipe || true
fi
${cmd} > $pipe
'';
};
in in
{ {
@ -65,59 +113,59 @@ in
extraGroups = [ "r-ryantm" ]; extraGroups = [ "r-ryantm" ];
}; };
systemd.services.nixpkgs-update-repology = mkNixpkgsUpdateService "repology" // { systemd.services.nixpkgs-update-delete-done = {
script = '' startAt = "daily";
${nixpkgs-update-bin} delete-done --delete after = [ "network.target" ];
${nixpkgs-update-bin} fetch-repology > /var/lib/nixpkgs-update/repology/packages-to-update-regular.txt wantedBy = [ "multi-user.target" ];
# reverse list restartTriggers = [ nixpkgs-update-bin ];
# sed '1!G;h;$!d' /var/lib/nixpkgs-update/repology/packages-to-update-regular.txt > /var/lib/nixpkgs-update/repology/packages-to-update.txt description = "nixpkgs-update delete done branches";
# skip reversing for now restartIfChanged = true;
cp /var/lib/nixpkgs-update/repology/packages-to-update-regular.txt /var/lib/nixpkgs-update/repology/packages-to-update.txt path = nixpkgsUpdateSystemDependencies;
${nixpkgs-update-command} environment.XDG_CONFIG_HOME = "/var/lib/nixpkgs-update/worker";
''; environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/worker";
environment.XDG_RUNTIME_DIR = "/run/nixpkgs-update/worker"; # for nix-update update scripts
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
Restart = "on-abort";
RestartSec = "5s";
WorkingDirectory = "/var/lib/nixpkgs-update/worker";
StateDirectory = "nixpkgs-update/worker";
StateDirectoryMode = "700";
CacheDirectory = "nixpkgs-update/worker";
CacheDirectoryMode = "700";
LogsDirectory = "nixpkgs-update/";
LogsDirectoryMode = "755";
RuntimeDirectory = "nixpkgs-update/worker";
RuntimeDirectoryMode = "700";
StandardOutput = "journal";
};
script = "${nixpkgs-update-bin} delete-done --delete";
}; };
systemd.services.nixpkgs-update-github = mkNixpkgsUpdateService "github" // { systemd.services.nixpkgs-update-fetch-repology = mkFetcher "${nixpkgs-update-bin} fetch-repology";
script = '' systemd.services.nixpkgs-update-fetch-updatescript = mkFetcher "${pkgs.nixUnstable}/bin/nix eval --raw -f ${./packages-with-update-script.nix}";
${nixpkgs-update-bin} delete-done --delete systemd.services.nixpkgs-update-fetch-pypi = mkFetcher "grep -rl $XDG_CACHE_HOME/nixpkgs -e buildPython | grep default | ${nixpkgs-update-pypi-releases'} --nixpkgs=/var/cache/nixpkgs-update/fetcher/nixpkgs";
${nixpkgs-update-github-releases'} > /var/lib/nixpkgs-update/github/packages-to-update.txt systemd.services.nixpkgs-update-fetch-github = mkFetcher nixpkgs-update-github-releases';
${nixpkgs-update-command}
'';
};
systemd.services.nixpkgs-update-pypi = mkNixpkgsUpdateService "pypi" // {
script = ''
${nixpkgs-update-bin} delete-done --delete
grep -rl $XDG_CACHE_HOME/nixpkgs -e buildPython | grep default | \
${nixpkgs-update-pypi-releases'} --nixpkgs=/var/cache/nixpkgs-update/pypi/nixpkgs > /var/lib/nixpkgs-update/pypi/packages-to-update.txt
${nixpkgs-update-command}
'';
};
systemd.services.nixpkgs-update-updatescript = mkNixpkgsUpdateService "updatescript" // { systemd.services.nixpkgs-update-worker1 = mkWorker "worker1";
script = '' systemd.services.nixpkgs-update-worker2 = mkWorker "worker2";
${nixpkgs-update-bin} delete-done --delete systemd.services.nixpkgs-update-worker3 = mkWorker "worker3";
${pkgs.nixUnstable}/bin/nix eval --raw -f ${./packages-with-update-script.nix} > /var/lib/nixpkgs-update/updatescript/packages-to-update.txt systemd.services.nixpkgs-update-worker4 = mkWorker "worker4";
${nixpkgs-update-bin} update-list --pr --outpaths --nixpkgs-review --attrpath
'';
};
systemd.tmpfiles.rules = [ systemd.tmpfiles.rules = [
"L /home/r-ryantm/.gitconfig - - - - ${./gitconfig.txt}" "L /home/r-ryantm/.gitconfig - - - - ${./gitconfig.txt}"
"d /home/r-ryantm/.ssh 700 r-ryantm r-ryantm - -" "d /home/r-ryantm/.ssh 700 r-ryantm r-ryantm - -"
"e /var/cache/nixpkgs-update/repology/nixpkgs-review - - - 1d -"
"e /var/cache/nixpkgs-update/github/nixpkgs-review - - - 1d -"
"e /var/cache/nixpkgs-update/pypi/nixpkgs-review - - - 1d -"
"e /var/cache/nixpkgs-update/updatescript/nixpkgs-review - - - 1d -"
"L /var/lib/nixpkgs-update/repology/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/github/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/pypi/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/updatescript/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/repology/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}" "e /var/cache/nixpkgs-update/worker/nixpkgs-review - - - 1d -"
"L /var/lib/nixpkgs-update/github/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}"
"L /var/lib/nixpkgs-update/pypi/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}" "L /var/lib/nixpkgs-update/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}" "L /var/lib/nixpkgs-update/worker/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/updatescript/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}" ]; "L /var/lib/nixpkgs-update/worker/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}"
];
sops.secrets.github-r-ryantm-key = { sops.secrets.github-r-ryantm-key = {
path = "/home/r-ryantm/.ssh/id_rsa"; path = "/home/r-ryantm/.ssh/id_rsa";

6
flake.lock generated
View file

@ -201,11 +201,11 @@
"nixpkgs": "nixpkgs_4" "nixpkgs": "nixpkgs_4"
}, },
"locked": { "locked": {
"lastModified": 1658974277, "lastModified": 1658978650,
"narHash": "sha256-H0/k4lvr3RVi1CUzGAriLukEm5hMC+EHwGhHmTTEfuI=", "narHash": "sha256-8kPAnKLTG4HAwVrT64B0zE3DTwKVhVlA4AxG8G7ajxg=",
"owner": "ryantm", "owner": "ryantm",
"repo": "nixpkgs-update", "repo": "nixpkgs-update",
"rev": "07bf5b44d65d6cac712a399159c7d2c1a987c6f9", "rev": "6a32dda818b3929aa7277efb51f7171f007cf47a",
"type": "github" "type": "github"
}, },
"original": { "original": {