From ca0f1c1ab514aeac451a9a929a6096939059e229 Mon Sep 17 00:00:00 2001
From: zowoq <59103226+zowoq@users.noreply.github.com>
Date: Sat, 19 Oct 2024 14:40:56 +1000
Subject: [PATCH] nixpkgs-update: move worker to bash script
---
hosts/build02/nixpkgs-update.nix | 54 +-------------------------------
hosts/build02/worker.bash | 51 ++++++++++++++++++++++++++++++
2 files changed, 52 insertions(+), 53 deletions(-)
create mode 100644 hosts/build02/worker.bash
diff --git a/hosts/build02/nixpkgs-update.nix b/hosts/build02/nixpkgs-update.nix
index f106249..8e9db21 100644
--- a/hosts/build02/nixpkgs-update.nix
+++ b/hosts/build02/nixpkgs-update.nix
@@ -56,59 +56,7 @@ let
StandardOutput = "journal";
};
- script = ''
- mkdir -p "$LOGS_DIRECTORY/~workers/"
- # This is for public logs at nixpkgs-update-logs.nix-community.org/~workers
- exec > >(rotatelogs -eD "$LOGS_DIRECTORY"'/~workers/%Y-%m-%d-${name}.stdout.log' 86400)
- exec 2> >(rotatelogs -eD "$LOGS_DIRECTORY"'/~workers/%Y-%m-%d-${name}.stderr.log' 86400 >&2)
-
- socket=/run/nixpkgs-update-supervisor/work.sock
-
- function run-nixpkgs-update {
- exit_code=0
- set -x
- timeout 6h ${nixpkgs-update-bin} update-batch --pr --outpaths --nixpkgs-review "$attr_path $payload" || exit_code=$?
- set +x
- if [ $exit_code -eq 124 ]; then
- echo "Update was interrupted because it was taking too long."
- fi
- msg="DONE $attr_path $exit_code"
- }
-
- msg=READY
- while true; do
- response=$(echo "$msg" | socat -t5 UNIX-CONNECT:"$socket" - || true)
- case "$response" in
- "") # connection error; retry
- sleep 5
- ;;
- NOJOBS)
- msg=READY
- sleep 60
- ;;
- JOB\ *)
- read -r attr_path payload <<< "''${response#JOB }"
- # If one worker is initializing the nixpkgs clone, the other will
- # try to use the incomplete clone, consuming a bunch of jobs and
- # throwing them away. So we use a crude locking mechanism to
- # run only one worker when there isn't a nixpkgs directory yet.
- # Once the directory exists and this initial lock is released,
- # multiple workers can run concurrently.
- lockdir="$XDG_CACHE_HOME/.nixpkgs.lock"
- if [ ! -e "$XDG_CACHE_HOME/nixpkgs" ] && mkdir "$lockdir"; then
- trap 'rmdir "$lockdir"' EXIT
- run-nixpkgs-update
- rmdir "$lockdir"
- trap - EXIT
- continue
- fi
- while [ -e "$lockdir" ]; do
- sleep 10
- done
- run-nixpkgs-update
- esac
- done
- '';
+ script = builtins.readFile ./worker.bash;
};
mkFetcher = name: cmd: {
diff --git a/hosts/build02/worker.bash b/hosts/build02/worker.bash
new file mode 100644
index 0000000..d314e1e
--- /dev/null
+++ b/hosts/build02/worker.bash
@@ -0,0 +1,51 @@
+mkdir -p "$LOGS_DIRECTORY/~workers/"
+# This is for public logs at nixpkgs-update-logs.nix-community.org/~workers
+exec > >(rotatelogs -eD "$LOGS_DIRECTORY"'/~workers/%Y-%m-%d-${name}.stdout.log' 86400)
+exec 2> >(rotatelogs -eD "$LOGS_DIRECTORY"'/~workers/%Y-%m-%d-${name}.stderr.log' 86400 >&2)
+
+socket=/run/nixpkgs-update-supervisor/work.sock
+
+function run-nixpkgs-update {
+ exit_code=0
+ set -x
+ timeout 6h ${nixpkgs-update-bin} update-batch --pr --outpaths --nixpkgs-review "$attr_path $payload" || exit_code=$?
+ set +x
+ if [ $exit_code -eq 124 ]; then
+ echo "Update was interrupted because it was taking too long."
+ fi
+ msg="DONE $attr_path $exit_code"
+}
+
+msg=READY
+while true; do
+ response=$(echo "$msg" | socat -t5 UNIX-CONNECT:"$socket" - || true)
+ case "$response" in
+ "") # connection error; retry
+ sleep 5
+ ;;
+ NOJOBS)
+ msg=READY
+ sleep 60
+ ;;
+ JOB\ *)
+ read -r attr_path payload <<< "''${response#JOB }"
+ # If one worker is initializing the nixpkgs clone, the other will
+ # try to use the incomplete clone, consuming a bunch of jobs and
+ # throwing them away. So we use a crude locking mechanism to
+ # run only one worker when there isn't a nixpkgs directory yet.
+ # Once the directory exists and this initial lock is released,
+ # multiple workers can run concurrently.
+ lockdir="$XDG_CACHE_HOME/.nixpkgs.lock"
+ if [ ! -e "$XDG_CACHE_HOME/nixpkgs" ] && mkdir "$lockdir"; then
+ trap 'rmdir "$lockdir"' EXIT
+ run-nixpkgs-update
+ rmdir "$lockdir"
+ trap - EXIT
+ continue
+ fi
+ while [ -e "$lockdir" ]; do
+ sleep 10
+ done
+ run-nixpkgs-update
+ esac
+done