diff --git a/.maintenance/clone-all-repositories.sh b/.maintenance/clone-all-repositories.sh index 33bf6b7..60328a8 100755 --- a/.maintenance/clone-all-repositories.sh +++ b/.maintenance/clone-all-repositories.sh @@ -3,6 +3,29 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export SCRIPT_DIR + +# peek.php renders a live lane of output per parallel job. Its use is +# optional: it degrades to a plain passthrough on platforms that cannot +# support the display (e.g. Windows) and can be disabled with NO_PEEK=1. +PEEK_PHP="${SCRIPT_DIR}/peek.php" +export PEEK_PHP + +# Parallel jobs run behind the peek display, where an interactive prompt is +# instantly overdrawn and would hang the run waiting for input nobody can +# see. Git and ssh prompt on /dev/tty (not stdin), so disable prompting +# entirely: failures then surface as visible errors in the job's lane. +# - GIT_TERMINAL_PROMPT=0: no credential/username prompts from git itself. +# - BatchMode=yes: ssh fails instead of asking for passphrases or host key +# confirmation (keys served by an ssh-agent keep working). Only set when +# GIT_SSH_COMMAND is not already customized. +# - GIT_MERGE_AUTOEDIT=no: a non-fast-forward pull keeps the default merge +# message instead of opening an editor. +export GIT_TERMINAL_PROMPT=0 +export GIT_MERGE_AUTOEDIT=no +if [[ -z "${GIT_SSH_COMMAND:-}" ]]; then + export GIT_SSH_COMMAND="ssh -oBatchMode=yes" +fi if ! command -v jq &>/dev/null; then echo "Required command 'jq' is not installed or not available in PATH." >&2 @@ -80,8 +103,11 @@ get_destination() { fi } -CLONE_LIST=() -UPDATE_FOLDERS=() +# One task per repository: sync-repository.sh clones missing folders and +# refreshes existing ones. Running a single parallel pass over all +# repositories keeps all ${CORES} slots busy for the whole run, instead of +# a clone stage and a refresh stage separated by a barrier. +TASK_LIST=() while IFS=$'\t' read -r name clone_url ssh_url; do if is_skipped "${name}"; then @@ -90,21 +116,14 @@ while IFS=$'\t' read -r name clone_url ssh_url; do destination=$(get_destination "${name}") - if [[ ! -d "${destination}" ]]; then - if [[ -n "${GITHUB_ACTION:-}" ]]; then - CLONE_LIST+=("${destination}"$'\t'"${clone_url}") - else - CLONE_LIST+=("${destination}"$'\t'"${ssh_url}") - fi + if [[ -n "${GITHUB_ACTION:-}" ]]; then + TASK_LIST+=("${destination}"$'\t'"${clone_url}") + else + TASK_LIST+=("${destination}"$'\t'"${ssh_url}") fi - - UPDATE_FOLDERS+=("${destination}") done < <(echo "${RESPONSE}" | jq -r '.[] | [.name, .clone_url, .ssh_url] | @tsv') -if [[ ${#CLONE_LIST[@]} -gt 0 ]]; then - printf '%s\n' "${CLONE_LIST[@]}" | xargs -n2 -P"${CORES}" bash "${SCRIPT_DIR}/clone-repository.sh" -fi - -if [[ ${#UPDATE_FOLDERS[@]} -gt 0 ]]; then - printf '%s\n' "${UPDATE_FOLDERS[@]}" | xargs -P"${CORES}" -I% php "${SCRIPT_DIR}/refresh-repository.php" % +if [[ ${#TASK_LIST[@]} -gt 0 ]]; then + printf '%s\n' "${TASK_LIST[@]}" | php "${PEEK_PHP}" -- xargs -n2 -P"${CORES}" \ + bash -c 'exec php "${PEEK_PHP}" run -n "$1" -- bash "${SCRIPT_DIR}/sync-repository.sh" "$1" "$2"' _ fi diff --git a/.maintenance/peek.php b/.maintenance/peek.php new file mode 100755 index 0000000..ec99b9c --- /dev/null +++ b/.maintenance/peek.php @@ -0,0 +1,672 @@ +#!/usr/bin/env php + 0 && ( $now - $at ) < 2.0 ) { + return $size; + } + $at = $now; + $cols = (int) getenv( 'COLUMNS' ); + $rows = (int) getenv( 'LINES' ); + if ( $cols <= 0 || $rows <= 0 ) { + $out = @shell_exec( 'stty size < /dev/tty 2>/dev/null' ); + if ( $out && preg_match( '#^(\d+)\s+(\d+)#', trim( $out ), $m ) ) { + $rows = (int) $m[1]; + $cols = (int) $m[2]; + } + } + $size = array( $cols > 0 ? $cols : 80, $rows > 0 ? $rows : 24 ); + return $size; +} + +function peek_which( $bin ) { + if ( false !== strpos( $bin, '/' ) ) { + return is_executable( $bin ) ? $bin : null; + } + foreach ( explode( PATH_SEPARATOR, (string) getenv( 'PATH' ) ) as $dir ) { + if ( '' === $dir ) { + continue; + } + $candidate = $dir . '/' . $bin; + if ( is_file( $candidate ) && is_executable( $candidate ) ) { + return $candidate; + } + } + return null; +} + +/** + * Run a command unchanged, inheriting stdio, and return its exit code. + * Used whenever the display is unavailable so behavior stays identical. + */ +function peek_passthrough( array $cmd ) { + if ( function_exists( 'pcntl_exec' ) ) { + $bin = peek_which( $cmd[0] ); + if ( null !== $bin ) { + @pcntl_exec( $bin, array_slice( $cmd, 1 ) ); + // Falls through only if exec itself failed. + } + } + $spec = array( + 0 => STDIN, + 1 => STDOUT, + 2 => STDERR, + ); + if ( PHP_VERSION_ID >= 70400 ) { + $proc = @proc_open( $cmd, $spec, $pipes ); + } else { + $proc = @proc_open( implode( ' ', array_map( 'escapeshellarg', $cmd ) ), $spec, $pipes ); + } + if ( ! is_resource( $proc ) ) { + fwrite( STDERR, "peek: failed to run: {$cmd[0]}\n" ); + return 127; + } + return proc_close( $proc ); +} + +// -------------------------------------------------------------------------- +// client side: peek.php run / peek.php pipe +// -------------------------------------------------------------------------- + +/** + * Write side of the protocol. Never fatal: if the display is gone, we + * silently stop reporting rather than killing the job. + */ +class PeekFeed { + + private $sock = null; + private $lane; + + public function __construct( $lane, $name ) { + $this->lane = (string) $lane; + $path = getenv( 'PEEK_SOCK' ); + if ( $path ) { + $this->sock = @stream_socket_client( 'udg://' . $path, $errno, $errstr, 1 ); + if ( $this->sock ) { + $this->send( 'OPEN', $name ); + } + } + } + + public function send( $kind, $payload ) { + if ( ! $this->sock ) { + return; + } + $data = $kind . PEEK_SEP . $this->lane . PEEK_SEP . substr( $payload, 0, PEEK_MAXLINE ); + $sent = @stream_socket_sendto( $this->sock, $data ); + if ( false === $sent || $sent < 0 ) { + $this->sock = null; + } + } + + public function line( $raw ) { + $this->send( 'LINE', $raw ); + } + + public function close( $rc ) { + $this->send( 'EXIT', (string) $rc ); + } +} + +function peek_cmd_run( array $argv ) { + $name = null; + $rest = array(); + $help = false; + while ( $argv ) { + $arg = array_shift( $argv ); + if ( '-n' === $arg || '--name' === $arg ) { + $name = array_shift( $argv ); + } elseif ( '-h' === $arg || '--help' === $arg ) { + $help = true; + } elseif ( '--' === $arg ) { + $rest = $argv; + break; + } else { + $rest = array_merge( array( $arg ), $argv ); + break; + } + } + if ( $help || ! $rest ) { + fwrite( STDERR, "usage: peek.php run [-n NAME] -- COMMAND [ARGS...]\n" ); + return $help ? 0 : 2; + } + + // No display: become the command. This is what makes peek droppable + // into scripts that also run standalone. + if ( ! getenv( 'PEEK_SOCK' ) || ! peek_supported() ) { + return peek_passthrough( $rest ); + } + + $feed = new PeekFeed( getmypid(), null !== $name ? $name : implode( ' ', $rest ) ); + $spec = array( + 0 => STDIN, + 1 => array( 'pipe', 'w' ), + 2 => array( 'redirect', 1 ), + ); + $proc = @proc_open( $rest, $spec, $pipes ); + if ( ! is_resource( $proc ) ) { + $feed->close( 127 ); + fwrite( STDERR, "peek: failed to run: {$rest[0]}\n" ); + return 127; + } + while ( false !== ( $line = fgets( $pipes[1] ) ) ) { + $feed->line( rtrim( $line, "\n" ) ); + } + fclose( $pipes[1] ); + $rc = proc_close( $proc ); + $feed->close( $rc ); + return $rc; +} + +function peek_cmd_pipe( array $argv ) { + $name = 'stdin'; + while ( $argv ) { + $arg = array_shift( $argv ); + if ( '-n' === $arg || '--name' === $arg ) { + $name = array_shift( $argv ); + } + } + $feed = new PeekFeed( getmypid(), $name ); + while ( false !== ( $line = fgets( STDIN ) ) ) { + fwrite( STDOUT, $line ); // Stay a tee, so pipe mode drops into a pipeline. + fflush( STDOUT ); + $feed->line( rtrim( $line, "\n" ) ); + } + $feed->close( 0 ); + return 0; +} + +// -------------------------------------------------------------------------- +// server side: the display +// -------------------------------------------------------------------------- + +class PeekLane { + + public $name; + public $tail = array(); + public $start; + public $end = null; + public $rc = null; + public $committed = false; + + private $max; + + public function __construct( $name, $peek_lines ) { + $this->name = $name; + $this->max = max( 1, $peek_lines ); + $this->start = microtime( true ); + } + + public function add_line( $text ) { + $this->tail[] = $text; + if ( count( $this->tail ) > $this->max ) { + array_shift( $this->tail ); + } + } + + public function is_running() { + return null === $this->rc; + } + + public function header( $frame, $width, $color = true ) { + $spin = peek_spinner(); + if ( null === $this->rc ) { + $glyph = $spin[ $frame % count( $spin ) ]; + $tint = PEEK_CYAN; + $right = peek_dur( microtime( true ) - $this->start ); + $right_tint = PEEK_GREY; + } elseif ( 0 === $this->rc ) { + $glyph = '✔'; + $tint = PEEK_GREEN; + $right = peek_dur( $this->end - $this->start ); + $right_tint = PEEK_GREY; + } else { + $glyph = '✘'; + $tint = PEEK_RED; + $right = 'exit ' . $this->rc . ' · ' . peek_dur( $this->end - $this->start ); + $right_tint = PEEK_RED; + } + $right_len = peek_len( $right ); + $name_fit = peek_fit( $this->name, max( 0, $width - $right_len - 4 ) ); + $plain = $glyph . ' ' . $name_fit; + $pad = str_repeat( ' ', max( 1, $width - peek_len( $plain ) - $right_len ) ); + if ( ! $color ) { + return $plain . $pad . $right; + } + if ( null === $this->rc ) { + $styled_name = PEEK_BOLD . $name_fit . PEEK_RESET; + } elseif ( 0 === $this->rc ) { + $styled_name = $name_fit; + } else { + $styled_name = PEEK_RED . PEEK_BOLD . $name_fit . PEEK_RESET; + } + return $tint . PEEK_BOLD . $glyph . PEEK_RESET . ' ' . $styled_name + . $pad . $right_tint . $right . PEEK_RESET; + } +} + +class PeekDisplay { + + public $lanes = array(); + public $order = array(); + + private $peek_lines; + private $prev = 0; + private $frame = 0; + private $start; + + public function __construct( $peek_lines ) { + $this->peek_lines = $peek_lines; + $this->start = microtime( true ); + } + + public function event( $kind, $lane, $payload ) { + if ( 'OPEN' === $kind ) { + if ( ! isset( $this->lanes[ $lane ] ) ) { + $this->lanes[ $lane ] = new PeekLane( peek_clean( $payload ), $this->peek_lines ); + $this->order[] = $lane; + } + } elseif ( isset( $this->lanes[ $lane ] ) ) { + $ln = $this->lanes[ $lane ]; + if ( 'LINE' === $kind ) { + $text = peek_clean( $payload ); + if ( '' !== trim( $text ) ) { + $ln->add_line( $text ); + } + } elseif ( 'EXIT' === $kind ) { + $ln->rc = (int) $payload; + $ln->end = microtime( true ); + } + } + } + + public function counts() { + $running = 0; + $ok = 0; + $failed = 0; + foreach ( $this->order as $key ) { + $rc = $this->lanes[ $key ]->rc; + if ( null === $rc ) { + ++$running; + } elseif ( 0 === $rc ) { + ++$ok; + } else { + ++$failed; + } + } + return array( $running, $ok, $failed ); + } + + public function footer( $color = true ) { + list( $running, $ok, $failed ) = $this->counts(); + $elapsed = peek_dur( microtime( true ) - $this->start ); + if ( ! $color ) { + $bits = array(); + if ( $running ) { + $bits[] = $running . ' running'; + } + $bits[] = 'ok ' . $ok; + if ( $failed ) { + $bits[] = 'failed ' . $failed; + } + $bits[] = $elapsed; + return implode( ' · ', $bits ); + } + $sep = PEEK_GREY . ' · ' . PEEK_RESET; + $bits = array(); + if ( $running ) { + $bits[] = PEEK_CYAN . $running . ' running' . PEEK_RESET; + } + $bits[] = PEEK_GREEN . '✔ ' . $ok . PEEK_RESET; + if ( $failed ) { + $bits[] = PEEK_RED . PEEK_BOLD . '✘ ' . $failed . PEEK_RESET; + } + $bits[] = PEEK_GREY . $elapsed . PEEK_RESET; + return ' ' . PEEK_GREY . '─' . PEEK_RESET . ' ' . implode( $sep, $bits ); + } + + public function compose( $rows, $cols ) { + $commit = array(); + $live = array(); + $lanes = array(); + foreach ( $this->order as $key ) { + $lanes[] = $this->lanes[ $key ]; + } + foreach ( $lanes as $ln ) { + if ( null !== $ln->rc && ! $ln->committed ) { + $ln->committed = true; + $commit[] = $ln->header( $this->frame, $cols ); + // Keep the captured tail of a failed job on screen: it scrolls + // into history with the header, so the error context survives. + if ( 0 !== $ln->rc ) { + foreach ( $ln->tail as $text ) { + $commit[] = ' ' . PEEK_RED . '│' . PEEK_RESET . ' ' + . PEEK_GREY . peek_fit( $text, $cols - 4 ) . PEEK_RESET; + } + } + } + } + $pending = array(); + foreach ( $lanes as $ln ) { + if ( ! $ln->committed ) { + $pending[] = $ln; + } + } + $budget = max( 0, $rows - 3 ) - count( $pending ); + $active = array(); + foreach ( $pending as $ln ) { + if ( $ln->is_running() ) { + $active[] = $ln; + } + } + $share = ( $active && $budget > 0 ) + ? min( $this->peek_lines, intdiv( $budget, count( $active ) ) ) + : 0; + foreach ( $pending as $ln ) { + $live[] = $ln->header( $this->frame, $cols ); + if ( $ln->is_running() && $share ) { + foreach ( array_slice( $ln->tail, -$share ) as $text ) { + $live[] = ' ' . PEEK_GREY . '│' . PEEK_RESET . ' ' + . PEEK_DIM . peek_fit( $text, $cols - 4 ) . PEEK_RESET; + } + } + } + $live[] = $this->footer(); + return array( $commit, $live ); + } + + public function draw( $out ) { + list( $cols, $rows ) = peek_term_size(); + list( $commit, $live ) = $this->compose( $rows, $cols ); + $buf = $this->prev ? "\033[" . $this->prev . 'A' : ''; + foreach ( array_merge( $commit, $live ) as $line ) { + $buf .= "\033[2K" . $line . "\n"; + } + $buf .= "\033[J"; + fwrite( $out, $buf ); + fflush( $out ); + $this->prev = count( $live ); + ++$this->frame; + } + + public function summary() { + $lines = array(); + foreach ( $this->order as $key ) { + $lines[] = $this->lanes[ $key ]->header( 0, 80, false ); + } + $lines[] = $this->footer( false ); + return $lines; + } +} + +function peek_drain( $srv, PeekDisplay $disp ) { + while ( true ) { + $data = @stream_socket_recvfrom( $srv, 131072 ); + if ( false === $data || '' === $data || null === $data ) { + return; + } + $parts = explode( PEEK_SEP, $data, 3 ); + if ( 3 === count( $parts ) ) { + $disp->event( $parts[0], $parts[1], $parts[2] ); + } + } +} + +function peek_cmd_serve( array $argv ) { + $peek_lines = 6; + $fps = 12.5; + $help = false; + $rest = array(); + while ( $argv ) { + $arg = array_shift( $argv ); + if ( '--peek' === $arg ) { + $peek_lines = (int) array_shift( $argv ); + } elseif ( 0 === strpos( $arg, '--peek=' ) ) { + $peek_lines = (int) substr( $arg, 7 ); + } elseif ( '--fps' === $arg ) { + $fps = (float) array_shift( $argv ); + } elseif ( 0 === strpos( $arg, '--fps=' ) ) { + $fps = (float) substr( $arg, 6 ); + } elseif ( '-h' === $arg || '--help' === $arg ) { + $help = true; + } elseif ( '--' === $arg ) { + $rest = $argv; + break; + } else { + $rest = array_merge( array( $arg ), $argv ); + break; + } + } + if ( $help || ! $rest ) { + fwrite( STDERR, "usage: peek.php [--peek N] [--fps F] -- COMMAND [ARGS...]\n" ); + fwrite( STDERR, " peek.php run [-n NAME] -- COMMAND [ARGS...]\n" ); + fwrite( STDERR, " peek.php pipe [-n NAME]\n" ); + return $help ? 0 : 2; + } + + if ( ! peek_supported() ) { + return peek_passthrough( $rest ); + } + + $tmp = sys_get_temp_dir() . '/peek.' . getmypid() . '.' . substr( md5( uniqid( '', true ) ), 0, 6 ); + if ( ! @mkdir( $tmp, 0700, true ) ) { + return peek_passthrough( $rest ); + } + $sock_path = $tmp . '/sock'; + $srv = @stream_socket_server( 'udg://' . $sock_path, $errno, $errstr, STREAM_SERVER_BIND ); + if ( ! $srv ) { + @rmdir( $tmp ); + return peek_passthrough( $rest ); + } + stream_set_blocking( $srv, false ); + + $env = getenv(); + $env['PEEK_SOCK'] = $sock_path; + + $tty = function_exists( 'stream_isatty' ) && @stream_isatty( STDOUT ); + + // The driver's own output would fight the live region, so hold it back + // and replay it once the display tears down. + $spec = $tty + ? array( + 0 => STDIN, + 1 => array( 'pipe', 'w' ), + 2 => array( 'redirect', 1 ), + ) + : array( + 0 => STDIN, + 1 => STDOUT, + 2 => STDERR, + ); + $proc = @proc_open( $rest, $spec, $pipes, null, $env ); + if ( ! is_resource( $proc ) ) { + fclose( $srv ); + @unlink( $sock_path ); + @rmdir( $tmp ); + fwrite( STDERR, "peek: failed to run: {$rest[0]}\n" ); + return 127; + } + + $disp = new PeekDisplay( $peek_lines ); + $held = ''; + $restore = function () use ( $tty ) { + if ( $tty ) { + fwrite( STDOUT, "\033[?25h" ); + fflush( STDOUT ); + } + }; + if ( $tty ) { + fwrite( STDOUT, "\033[?25l" ); + register_shutdown_function( $restore ); + if ( function_exists( 'pcntl_async_signals' ) ) { + pcntl_async_signals( true ); + $on_signal = function () use ( $restore ) { + $restore(); + exit( 130 ); + }; + pcntl_signal( SIGINT, $on_signal ); + pcntl_signal( SIGTERM, $on_signal ); + } + stream_set_blocking( $pipes[1], false ); + } + + $frame_us = (int) ( 1000000 / max( $fps, 1 ) ); + $exit = null; + while ( true ) { + $status = proc_get_status( $proc ); + if ( ! $status['running'] && null === $exit ) { + $exit = $status['exitcode']; + } + $read = array( $srv ); + if ( $tty && is_resource( $pipes[1] ) && ! feof( $pipes[1] ) ) { + $read[] = $pipes[1]; + } + $write = null; + $except = null; + @stream_select( $read, $write, $except, 0, $frame_us ); + peek_drain( $srv, $disp ); + if ( $tty && is_resource( $pipes[1] ) ) { + while ( false !== ( $chunk = fread( $pipes[1], 65536 ) ) && '' !== $chunk ) { + $held .= $chunk; + } + } + if ( $tty ) { + $disp->draw( STDOUT ); + } + if ( null !== $exit ) { + break; + } + } + + usleep( 250000 ); // Drain late EXIT datagrams. + peek_drain( $srv, $disp ); + if ( $tty ) { + while ( is_resource( $pipes[1] ) && false !== ( $chunk = fread( $pipes[1], 65536 ) ) && '' !== $chunk ) { + $held .= $chunk; + } + $disp->draw( STDOUT ); + fclose( $pipes[1] ); + } + proc_close( $proc ); + fclose( $srv ); + @unlink( $sock_path ); + @rmdir( $tmp ); + $restore(); + + if ( $tty ) { + if ( '' !== $held ) { + fwrite( STDOUT, $held ); + fflush( STDOUT ); + } + } else { + foreach ( $disp->summary() as $line ) { + fwrite( STDOUT, $line . "\n" ); + } + } + return null === $exit ? 0 : $exit; +} + +function peek_main( array $argv ) { + array_shift( $argv ); + if ( isset( $argv[0] ) && 'run' === $argv[0] ) { + return peek_cmd_run( array_slice( $argv, 1 ) ); + } + if ( isset( $argv[0] ) && 'pipe' === $argv[0] ) { + return peek_cmd_pipe( array_slice( $argv, 1 ) ); + } + return peek_cmd_serve( $argv ); +} + +exit( peek_main( $argv ) ); diff --git a/.maintenance/sync-repository.sh b/.maintenance/sync-repository.sh new file mode 100755 index 0000000..dad050c --- /dev/null +++ b/.maintenance/sync-repository.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Bring a single repository up to date: clone it if the folder is missing, +# refresh it otherwise. Freshly cloned repositories are already current, so +# they skip the refresh. This lets the caller run one continuous parallel +# pass over all repositories instead of a clone stage and a refresh stage +# separated by a barrier. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [[ $# -lt 1 ]]; then + echo "Usage: sync-repository.sh []" >&2 + exit 1 +fi + +destination="$1" +clone_url="${2:-}" + +if [[ ! -d "${destination}" ]]; then + if [[ -z "${clone_url}" ]]; then + echo "Folder '${destination}' is missing and no clone URL was provided." >&2 + exit 1 + fi + exec bash "${SCRIPT_DIR}/clone-repository.sh" "${destination}" "${clone_url}" +fi + +exec php "${SCRIPT_DIR}/refresh-repository.php" "${destination}" diff --git a/phpcs.xml.dist b/phpcs.xml.dist index 58dbf40..a6a976b 100644 --- a/phpcs.xml.dist +++ b/phpcs.xml.dist @@ -52,4 +52,23 @@ */.maintenance/* + + + */.maintenance/peek.php + + + */.maintenance/peek.php + + + + + */.maintenance/peek.php + + + */.maintenance/peek.php + +