#!/bin/sh # Usage: ALERTS_URL=https://alerts.example.com/k/ alerts-run -- # Knocks start, runs the command, then knocks ok/fail and exits with the command's own exit code. # A knock that can't connect is tried again for about a minute (the monitor may be restarting); the # job doesn't wait for its start knock, but its ok/fail goes only after the start, so they pair up. # The note's first line is a summary for the phone alert, "exit N: " # (or just "exit N"), followed by the end of the output, all within the server's 2 KB note limit. # The job's error output is merged into its normal output. A failed knock never fails or reruns the job. usage() { echo "usage: ALERTS_URL=https://alerts.example.com/k/ alerts-run -- " >&2; exit 64; } [ $# -ge 1 ] || usage slug="$1"; shift [ "$1" = "--" ] && shift { [ -n "$slug" ] && [ $# -gt 0 ]; } || usage # A missing setting must not cost us the job itself: run it unreported (the check will alert as missed). if [ -z "$ALERTS_URL" ]; then echo "alerts-run: ALERTS_URL is not set, running $slug without reporting" >&2 exec "$@" fi url="${ALERTS_URL%/}/$slug" # Job containers often reuse the same process number, so add randomness (old form if /dev/urandom is missing). rand="$(od -An -N4 -tu4 /dev/urandom 2>/dev/null | tr -d ' \n')" run="$(date +%s)-$$${rand:+-$rand}" # The address holds the project's secret key, so errors name only the check and the knock, never the address. knock() { curl -fsS -m 10 --retry 5 --retry-connrefused --retry-delay 10 --retry-max-time 60 -o /dev/null --data-binary "@$2" "$url/$1?run=$run" 2>/dev/null || echo "alerts-run: could not reach the monitor to send $1 for $slug (curl exit $?)" >&2; } # If temp files can't be made, still run the job exactly once and keep its exit code. out="$(mktemp 2>/dev/null)" && rcf="$(mktemp 2>/dev/null)" || { echo "alerts-run: no temp dir, running without reporting" >&2; exec "$@"; } trap 'rm -f "$out" "$rcf" "$out.note"' EXIT knock start /dev/null & started=$! # ponytail: the whole output is buffered on disk before only the end is kept, so a very chatty # long job can fill a small /tmp; upgrade to a rolling tail (e.g. a fixed-size ring) if that bites. { "$@" 2>&1; echo $? > "$rcf"; } | tee "$out" rc="$(cat "$rcf")" # If the job's shell died before recording its exit code, report it as a failure. case "$rc" in ''|*[!0-9]*) rc=1 ;; esac # Last non-empty line (carriage returns from progress bars dropped, capped so one huge line can't eat the note). # Byte-wise (LC_ALL=C): the 2 KB cut can split a character, which some tools refuse in a UTF-8 locale. # A line cut at 200 loses its last partial word: a token cut short no longer looks like one, and would show. last="$(tail -c 2048 "$out" | LC_ALL=C tr -d '\r' | LC_ALL=C awk 'NF { l = $0 } END { s = substr(l, 1, 200); if (length(l) > 200) sub(/[^ ]*$/, "", s); sub(/ +$/, "", s); print s }')" if [ -n "$last" ]; then printf 'exit %s: %s\n' "$rc" "$last"; else printf 'exit %s\n' "$rc"; fi > "$out.note" # The server keeps the last 2048 bytes, so the summary plus the tail must fit or the summary gets cut. # The 8 spare bytes cover characters split by the cuts, which the server turns into 3-byte replacement marks. budget=$((2048 - 8 - $(wc -c < "$out.note"))) # A cut lands mid-line, maybe inside a secret's name ("Authorization: Bea|rer "): that whole # first partial line goes, even when it's all there is (minified JSON): hiding runs on the server, # after this cut, so text after a blind cut inside a line is never sent. The summary line still shows. if [ "$(wc -c < "$out")" -gt "$budget" ]; then tail -c "$budget" "$out" | sed 1d; else cat "$out"; fi >> "$out.note" wait "$started" if [ "$rc" -eq 0 ]; then knock ok "$out.note"; else knock fail "$out.note"; fi exit "$rc"