sercanto/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

nixpkgs-update: use fetcher -> fifo -> worker model

Browse source
This commit is contained in:
Ryan Mulligan 2022-07-27 19:34:53 -07:00
parent 75a9db784e
commit 37b2fbef90
2 changed files with 112 additions and 64 deletions

170
build02/nixpkgs-update.nix
View file

@ -21,39 +21,87 @@ let
nixpkgs-update-github-releases' = "${nixpkgs-update-github-releases}/main.py";
nixpkgs-update-pypi-releases' = "${nixpkgs-update-pypi-releases}/main.py";
mkNixpkgsUpdateService = name: {
mkWorker = name: {
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
restartTriggers = [ nixpkgs-update-bin ];
description = "nixpkgs-update ${name} service";
enable = true;
startAt = "daily";
restartIfChanged = false;
restartIfChanged = true;
path = nixpkgsUpdateSystemDependencies;
environment.XDG_CONFIG_HOME = "/var/lib/nixpkgs-update/${name}";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/${name}";
environment.XDG_RUNTIME_DIR = "/run/nixpkgs-update/${name}"; # for nix-update update scripts
# API_TOKEN is used by nixpkgs-update-github-releases
environment.API_TOKEN_FILE = "/var/lib/nixpkgs-update/github_token_with_username.txt";
# Used by nixpkgs-update-github-releases to install python dependencies
# Used by nixpkgs-update-pypi-releases
environment.NIX_PATH = "nixpkgs=/var/cache/nixpkgs-update/${name}/nixpkgs";
environment.XDG_CONFIG_HOME = "/var/lib/nixpkgs-update/worker";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/worker";
environment.XDG_RUNTIME_DIR = "/run/nixpkgs-update/worker"; # for nix-update update scripts
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
WorkingDirectory = "/var/lib/nixpkgs-update/${name}";
StateDirectory = "nixpkgs-update/${name}";
Restart = "on-abort";
RestartSec = "5s";
WorkingDirectory = "/var/lib/nixpkgs-update/worker";
StateDirectory = "nixpkgs-update/worker";
StateDirectoryMode = "700";
CacheDirectory = "nixpkgs-update/${name}";
CacheDirectory = "nixpkgs-update/worker";
CacheDirectoryMode = "700";
LogsDirectory = "nixpkgs-update/${name}";
LogsDirectory = "nixpkgs-update/";
LogsDirectoryMode = "755";
RuntimeDirectory = "nixpkgs-update/${name}";
RuntimeDirectory = "nixpkgs-update/worker";
RuntimeDirectoryMode = "700";
StandardOutput = "journal";
};
script = ''
pipe=/var/lib/nixpkgs-update/fifo
if [[ ! -p $pipe ]]; then
mkfifo $pipe || true
fi
exec 8<$pipe
while true
do
if read -u 8 line; then
set -x
${nixpkgs-update-bin} update-batch --pr --outpaths --nixpkgs-review "$line" || true
set +x
fi
done
'';
};
nixpkgs-update-command = "${nixpkgs-update-bin} update-list --pr --outpaths --nixpkgs-review";
mkFetcher = cmd: {
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
restartTriggers = [ nixpkgs-update-bin ];
path = nixpkgsUpdateSystemDependencies;
# API_TOKEN is used by nixpkgs-update-github-releases
environment.API_TOKEN_FILE = "/var/lib/nixpkgs-update/github_token_with_username.txt";
# Used by nixpkgs-update-github-releases to install python dependencies
# Used by nixpkgs-update-pypi-releases
environment.NIX_PATH = "nixpkgs=/var/cache/nixpkgs-update/fetcher/nixpkgs";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/fetcher/";
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
Restart = "on-abort";
RestartSec = "5s";
WorkingDirectory = "/var/lib/nixpkgs-update/";
StateDirectory = "nixpkgs-update";
StateDirectoryMode = "700";
};
script = ''
pipe=/var/lib/nixpkgs-update/fifo
if [[ ! -p $pipe ]]; then
mkfifo $pipe || true
fi
${cmd} > $pipe
'';
};
in
{
@ -65,59 +113,59 @@ in
extraGroups = [ "r-ryantm" ];
};
systemd.services.nixpkgs-update-repology = mkNixpkgsUpdateService "repology" // {
script = ''
${nixpkgs-update-bin} delete-done --delete
${nixpkgs-update-bin} fetch-repology > /var/lib/nixpkgs-update/repology/packages-to-update-regular.txt
# reverse list
# sed '1!G;h;$!d' /var/lib/nixpkgs-update/repology/packages-to-update-regular.txt > /var/lib/nixpkgs-update/repology/packages-to-update.txt
# skip reversing for now
cp /var/lib/nixpkgs-update/repology/packages-to-update-regular.txt /var/lib/nixpkgs-update/repology/packages-to-update.txt
${nixpkgs-update-command}
'';
systemd.services.nixpkgs-update-delete-done = {
startAt = "daily";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
restartTriggers = [ nixpkgs-update-bin ];
description = "nixpkgs-update delete done branches";
restartIfChanged = true;
path = nixpkgsUpdateSystemDependencies;
environment.XDG_CONFIG_HOME = "/var/lib/nixpkgs-update/worker";
environment.XDG_CACHE_HOME = "/var/cache/nixpkgs-update/worker";
environment.XDG_RUNTIME_DIR = "/run/nixpkgs-update/worker"; # for nix-update update scripts
serviceConfig = {
Type = "simple";
User = "r-ryantm";
Group = "r-ryantm";
Restart = "on-abort";
RestartSec = "5s";
WorkingDirectory = "/var/lib/nixpkgs-update/worker";
StateDirectory = "nixpkgs-update/worker";
StateDirectoryMode = "700";
CacheDirectory = "nixpkgs-update/worker";
CacheDirectoryMode = "700";
LogsDirectory = "nixpkgs-update/";
LogsDirectoryMode = "755";
RuntimeDirectory = "nixpkgs-update/worker";
RuntimeDirectoryMode = "700";
StandardOutput = "journal";
};
script = "${nixpkgs-update-bin} delete-done --delete";
};
systemd.services.nixpkgs-update-github = mkNixpkgsUpdateService "github" // {
script = ''
${nixpkgs-update-bin} delete-done --delete
${nixpkgs-update-github-releases'} > /var/lib/nixpkgs-update/github/packages-to-update.txt
${nixpkgs-update-command}
'';
};
systemd.services.nixpkgs-update-fetch-repology = mkFetcher "${nixpkgs-update-bin} fetch-repology";
systemd.services.nixpkgs-update-fetch-updatescript = mkFetcher "${pkgs.nixUnstable}/bin/nix eval --raw -f ${./packages-with-update-script.nix}";
systemd.services.nixpkgs-update-fetch-pypi = mkFetcher "grep -rl $XDG_CACHE_HOME/nixpkgs -e buildPython | grep default | ${nixpkgs-update-pypi-releases'} --nixpkgs=/var/cache/nixpkgs-update/fetcher/nixpkgs";
systemd.services.nixpkgs-update-fetch-github = mkFetcher nixpkgs-update-github-releases';
systemd.services.nixpkgs-update-pypi = mkNixpkgsUpdateService "pypi" // {
script = ''
${nixpkgs-update-bin} delete-done --delete
grep -rl $XDG_CACHE_HOME/nixpkgs -e buildPython | grep default | \
${nixpkgs-update-pypi-releases'} --nixpkgs=/var/cache/nixpkgs-update/pypi/nixpkgs > /var/lib/nixpkgs-update/pypi/packages-to-update.txt
${nixpkgs-update-command}
'';
};
systemd.services.nixpkgs-update-updatescript = mkNixpkgsUpdateService "updatescript" // {
script = ''
${nixpkgs-update-bin} delete-done --delete
${pkgs.nixUnstable}/bin/nix eval --raw -f ${./packages-with-update-script.nix} > /var/lib/nixpkgs-update/updatescript/packages-to-update.txt
${nixpkgs-update-bin} update-list --pr --outpaths --nixpkgs-review --attrpath
'';
};
systemd.services.nixpkgs-update-worker1 = mkWorker "worker1";
systemd.services.nixpkgs-update-worker2 = mkWorker "worker2";
systemd.services.nixpkgs-update-worker3 = mkWorker "worker3";
systemd.services.nixpkgs-update-worker4 = mkWorker "worker4";
systemd.tmpfiles.rules = [
"L /home/r-ryantm/.gitconfig - - - - ${./gitconfig.txt}"
"d /home/r-ryantm/.ssh 700 r-ryantm r-ryantm - -"
"e /var/cache/nixpkgs-update/repology/nixpkgs-review - - - 1d -"
"e /var/cache/nixpkgs-update/github/nixpkgs-review - - - 1d -"
"e /var/cache/nixpkgs-update/pypi/nixpkgs-review - - - 1d -"
"e /var/cache/nixpkgs-update/updatescript/nixpkgs-review - - - 1d -"
"L /var/lib/nixpkgs-update/repology/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/github/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/pypi/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/updatescript/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/repology/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}"
"L /var/lib/nixpkgs-update/github/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}"
"L /var/lib/nixpkgs-update/pypi/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}"
"L /var/lib/nixpkgs-update/updatescript/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}" ];
"e /var/cache/nixpkgs-update/worker/nixpkgs-review - - - 1d -"
"L /var/lib/nixpkgs-update/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}" "L /var/lib/nixpkgs-update/worker/github_token.txt - - - - ${config.sops.secrets.github-r-ryantm-token.path}"
"L /var/lib/nixpkgs-update/worker/cachix/cachix.dhall - - - - ${config.sops.secrets.nix-community-cachix.path}"
];
sops.secrets.github-r-ryantm-key = {
path = "/home/r-ryantm/.ssh/id_rsa";

6
flake.lock generated
View file

@ -201,11 +201,11 @@
"nixpkgs": "nixpkgs_4"
},
"locked": {
"lastModified": 1658974277,
"narHash": "sha256-H0/k4lvr3RVi1CUzGAriLukEm5hMC+EHwGhHmTTEfuI=",
"lastModified": 1658978650,
"narHash": "sha256-8kPAnKLTG4HAwVrT64B0zE3DTwKVhVlA4AxG8G7ajxg=",
"owner": "ryantm",
"repo": "nixpkgs-update",
"rev": "07bf5b44d65d6cac712a399159c7d2c1a987c6f9",
"rev": "6a32dda818b3929aa7277efb51f7171f007cf47a",
"type": "github"
},
"original": {