Skip to content

Data Refresh

Data Refresh #104

Workflow file for this run

name: Data Refresh
# This workflow is intentionally self-hosted only.
# Do not run scraping or enrichment refreshes on GitHub-hosted runners:
# session state, proxy behavior, and anti-bot handling are materially more reliable
# on controlled infrastructure than on ephemeral public CI.
on:
workflow_dispatch:
inputs:
profile:
description: Refresh profile to run
required: true
default: balanced
type: choice
options:
- balanced
- backfill
- sweep
refresh_list:
description: Optional slug, source URL, or source path to refresh
required: false
type: string
refresh_force:
description: Force raw source refresh even if the raw refresh window has not expired
required: false
default: false
type: boolean
skip_photos:
description: Skip refreshing missing local photo derivatives during the rebuild step
required: false
default: false
type: boolean
refresh_workers:
description: Max parallel workers for refresh, enrichment, and photo jobs
required: false
default: "4"
type: string
soft_timeout_minutes:
description: Stop the refresh command after this many minutes so partial changes can still be committed
required: false
default: "150"
type: string
schedule:
- cron: "17 3 * * *"
- cron: "43 5 1 * *"
permissions:
contents: write
pull-requests: write
concurrency:
group: data-refresh
cancel-in-progress: false
jobs:
refresh:
# Upstream default stays generic. Site repos can set the GitHub Actions
# repository variable DATA_REFRESH_RUNNER_LABELS to a JSON array such as
# ["self-hosted","Linux","residential"] to pin private runner pools.
runs-on: ${{ fromJSON(vars.DATA_REFRESH_RUNNER_LABELS || '["self-hosted","Linux"]') }}
timeout-minutes: 180
env:
REFRESH_BRANCH_PREFIX: automation/data-refresh
GOOGLE_PLACES_API_KEY: ${{ secrets.GOOGLE_PLACES_API_KEY }}
GMAPS_SCRAPER_PROXY: ${{ secrets.GMAPS_SCRAPER_PROXY }}
steps:
- name: Record job budget start
id: budget
shell: bash
run: echo "started_at_epoch=$(date +%s)" >> "${GITHUB_OUTPUT}"
- name: Check out repository
uses: actions/checkout@v5
with:
fetch-depth: 0
ref: main
token: ${{ secrets.GH_AUTOMATION_TOKEN != '' && secrets.GH_AUTOMATION_TOKEN || github.token }}
- name: Verify runner prerequisites
shell: bash
run: |
set -euo pipefail
if ! command -v unzip >/dev/null 2>&1; then
echo "::error::Missing required runner tool: unzip. Provision it on the self-hosted runner; do not install it inside this workflow."
exit 1
fi
if ! command -v timeout >/dev/null 2>&1; then
echo "::error::Missing required runner tool: timeout. Provision GNU coreutils on the self-hosted runner; do not install it inside this workflow."
exit 1
fi
required_libs=(
libnspr4.so
libnss3.so
libatk-1.0.so.0
libatk-bridge-2.0.so.0
libgbm.so.1
libglib-2.0.so.0
libgtk-3.so.0
libX11.so.6
libX11-xcb.so.1
libxcb.so.1
libXcomposite.so.1
libXdamage.so.1
libXext.so.6
libXfixes.so.3
libxkbcommon.so.0
libXrandr.so.2
)
LDCONFIG="$(command -v ldconfig || true)"
if [[ -z "${LDCONFIG}" ]]; then
for candidate in /sbin/ldconfig /usr/sbin/ldconfig; do
if [[ -x "${candidate}" ]]; then
LDCONFIG="${candidate}"
break
fi
done
fi
if [[ -z "${LDCONFIG}" ]]; then
echo "::error::ldconfig is not available on the runner, so shared library prerequisites cannot be verified."
exit 1
fi
ldconfig_output="$("${LDCONFIG}" -p)"
missing_libs=()
for library in "${required_libs[@]}"; do
if ! grep -Fq "${library}" <<< "${ldconfig_output}"; then
missing_libs+=("${library}")
fi
done
if (( ${#missing_libs[@]} > 0 )); then
printf '::error::Missing required browser libraries for CloakBrowser/Playwright: %s\n' "${missing_libs[*]}"
echo "::error::Provision the self-hosted runner with Playwright Chromium system dependencies, for example the distro-equivalent of 'playwright install-deps chromium'."
exit 1
fi
- name: Resolve refresh settings
id: settings
shell: bash
env:
PROFILE_INPUT: ${{ inputs.profile }}
REFRESH_LIST_INPUT: ${{ inputs.refresh_list }}
REFRESH_FORCE_INPUT: ${{ inputs.refresh_force }}
SKIP_PHOTOS_INPUT: ${{ inputs.skip_photos }}
REFRESH_WORKERS_INPUT: ${{ inputs.refresh_workers }}
SOFT_TIMEOUT_MINUTES_INPUT: ${{ inputs.soft_timeout_minutes }}
run: |
set -euo pipefail
if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
profile="${PROFILE_INPUT}"
refresh_list="${REFRESH_LIST_INPUT}"
refresh_force="${REFRESH_FORCE_INPUT}"
skip_photos="${SKIP_PHOTOS_INPUT}"
refresh_workers="${REFRESH_WORKERS_INPUT}"
soft_timeout_minutes="${SOFT_TIMEOUT_MINUTES_INPUT}"
else
case "${{ github.event.schedule }}" in
"43 5 1 * *")
profile="sweep"
;;
*)
profile="balanced"
;;
esac
refresh_list=""
refresh_force="false"
skip_photos="false"
refresh_workers="4"
soft_timeout_minutes="150"
fi
# Keep these aligned with this job's `timeout-minutes` and the
# refresh step's `timeout --kill-after` value. GitHub Actions does
# not expose `timeout-minutes` inside a run step.
job_timeout_minutes=180
pre_refresh_setup_buffer_minutes=15
kill_after_minutes=5
post_timeout_buffer_minutes=10
max_soft_timeout_minutes=$((job_timeout_minutes - pre_refresh_setup_buffer_minutes - kill_after_minutes - post_timeout_buffer_minutes))
if ! [[ "${soft_timeout_minutes}" =~ ^[0-9]+$ ]] || (( soft_timeout_minutes < 1 )); then
echo "::error::soft_timeout_minutes must be a positive integer."
exit 1
fi
if (( soft_timeout_minutes > max_soft_timeout_minutes )); then
echo "::error::soft_timeout_minutes must be <= ${max_soft_timeout_minutes} so the refresh can stop, summarize, commit, and open a PR before the ${job_timeout_minutes}-minute job timeout."
exit 1
fi
refresh_date="$(date -u +%Y-%m-%d)"
refresh_branch="${REFRESH_BRANCH_PREFIX}-${profile}"
{
echo "profile=${profile}"
echo "refresh_branch=${refresh_branch}"
echo "refresh_list=${refresh_list}"
echo "refresh_force=${refresh_force}"
echo "skip_photos=${skip_photos}"
echo "refresh_workers=${refresh_workers}"
echo "soft_timeout_minutes=${soft_timeout_minutes}"
echo "refresh_date=${refresh_date}"
} >> "${GITHUB_OUTPUT}"
- name: Set up Bun
uses: oven-sh/setup-bun@v2
with:
bun-version: 1.3.12
- name: Set up Node.js
uses: actions/setup-node@v5
with:
node-version: 22
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.14"
- name: Set up uv
uses: astral-sh/setup-uv@v8.1.0
with:
version: "0.11.6"
enable-cache: true
- name: Install frontend dependencies
run: bun ci
- name: Install Python dependencies
run: uv sync --frozen
- name: Configure Git identity
shell: bash
run: |
set -euo pipefail
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
- name: Prepare refresh branch
shell: bash
env:
REFRESH_BRANCH: ${{ steps.settings.outputs.refresh_branch }}
run: |
set -euo pipefail
git fetch origin main
if git ls-remote --exit-code --heads origin "${REFRESH_BRANCH}" >/dev/null 2>&1; then
git fetch origin "${REFRESH_BRANCH}:refs/remotes/origin/${REFRESH_BRANCH}"
fi
# Treat the refresh branch as disposable. Each run starts from the
# current main tip so old generated artifacts cannot conflict.
git switch -C "${REFRESH_BRANCH}" origin/main
- name: Run self-hosted refresh
id: refresh
shell: bash
env:
PROFILE: ${{ steps.settings.outputs.profile }}
REFRESH_LIST: ${{ steps.settings.outputs.refresh_list }}
REFRESH_FORCE: ${{ steps.settings.outputs.refresh_force }}
JOB_BUDGET_STARTED_AT_EPOCH: ${{ steps.budget.outputs.started_at_epoch }}
REFRESH_LOG_PATH: ${{ runner.temp }}/data-refresh.log
REFRESH_SUMMARY_PATH: ${{ runner.temp }}/data-refresh-summary.md
SKIP_PHOTOS: ${{ steps.settings.outputs.skip_photos }}
REFRESH_WORKERS: ${{ steps.settings.outputs.refresh_workers }}
SOFT_TIMEOUT_MINUTES: ${{ steps.settings.outputs.soft_timeout_minutes }}
run: |
set -euo pipefail
args=("${PROFILE}" "--refresh-workers" "${REFRESH_WORKERS}")
if [[ "${REFRESH_FORCE}" == "true" ]]; then
args+=("--refresh-force")
fi
if [[ "${SKIP_PHOTOS}" == "true" ]]; then
args+=("--skip-photos")
fi
if [[ -n "${REFRESH_LIST}" ]]; then
args+=("--refresh-list" "${REFRESH_LIST}")
fi
job_timeout_minutes=180
kill_after_minutes=5
post_timeout_buffer_minutes=10
setup_elapsed_seconds=$(($(date +%s) - JOB_BUDGET_STARTED_AT_EPOCH))
setup_elapsed_minutes=$(((setup_elapsed_seconds + 59) / 60))
max_effective_soft_timeout_minutes=$((job_timeout_minutes - setup_elapsed_minutes - kill_after_minutes - post_timeout_buffer_minutes))
if (( max_effective_soft_timeout_minutes < 1 )); then
echo "::error::Setup consumed too much of the ${job_timeout_minutes}-minute job timeout to leave room for refresh cleanup."
exit 1
fi
effective_soft_timeout_minutes="${SOFT_TIMEOUT_MINUTES}"
if (( effective_soft_timeout_minutes > max_effective_soft_timeout_minutes )); then
echo "::warning::Reducing refresh soft timeout from ${SOFT_TIMEOUT_MINUTES} to ${max_effective_soft_timeout_minutes} minutes because setup took ${setup_elapsed_minutes} minutes."
effective_soft_timeout_minutes="${max_effective_soft_timeout_minutes}"
fi
is_soft_timeout_status() {
local status="$1"
local elapsed_seconds="$2"
local soft_timeout_seconds=$((effective_soft_timeout_minutes * 60))
local kill_after_seconds=$((kill_after_minutes * 60))
[[ "${status}" == "124" ]] || {
[[ "${status}" == "137" ]] && (( elapsed_seconds >= soft_timeout_seconds + kill_after_seconds ))
}
}
echo "timed_out=false" >> "${GITHUB_OUTPUT}"
set +e
refresh_started_at="$(date +%s)"
timeout --signal=INT --kill-after=5m "${effective_soft_timeout_minutes}m" \
uv run python3 scripts/refresh_profiles.py "${args[@]}" | tee "${REFRESH_LOG_PATH}"
pipeline_status=("${PIPESTATUS[@]}")
refresh_finished_at="$(date +%s)"
set -e
refresh_status="${pipeline_status[0]}"
tee_status="${pipeline_status[1]}"
refresh_elapsed_seconds=$((refresh_finished_at - refresh_started_at))
if is_soft_timeout_status "${refresh_status}" "${refresh_elapsed_seconds}"; then
echo "::warning::Refresh stopped after ${effective_soft_timeout_minutes} minutes so partial progress can be committed."
{
echo ""
echo "Refresh stopped after ${effective_soft_timeout_minutes} minutes so partial progress can be committed."
} >> "${REFRESH_LOG_PATH}"
echo "timed_out=true" >> "${GITHUB_OUTPUT}"
elif (( tee_status != 0 )); then
echo "::error::tee failed while writing ${REFRESH_LOG_PATH}."
exit "${tee_status}"
elif (( refresh_status != 0 )); then
exit "${refresh_status}"
fi
uv run python3 scripts/refresh_summary.py --log "${REFRESH_LOG_PATH}" --output "${REFRESH_SUMMARY_PATH}"
- name: Create commit if data changed
id: commit
shell: bash
env:
PROFILE: ${{ steps.settings.outputs.profile }}
REFRESH_BRANCH: ${{ steps.settings.outputs.refresh_branch }}
REFRESH_TIMED_OUT: ${{ steps.refresh.outputs.timed_out }}
run: |
set -euo pipefail
site_dir="${FAVORITE_PLACES_SITE_DIR:-}"
if [[ -z "${site_dir}" ]]; then
if [[ -e site ]]; then
site_dir="site"
else
site_dir="site.example"
fi
fi
site_dir="${site_dir%/}"
stage_optional_path() {
local path="$1"
if [[ -e "${path}" ]] || git ls-files -- "${path}" | grep -q .; then
git add -A -- "${path}"
fi
}
cleanup_runtime_artifacts() {
local path="$1"
if [[ -d "${path}" ]]; then
find "${path}" \
\( -name "*.tmp" -o -name ".*.tmp" -o -name "*.tmp-journal" -o -name ".*.tmp-journal" \) \
-delete
fi
}
stage_cache=true
recover_sqlite_database() {
local db_path="$1"
local journal_path="${db_path}-journal"
if [[ ! -e "${journal_path}" ]]; then
return
fi
if [[ "${REFRESH_TIMED_OUT}" == "true" ]]; then
echo "::warning::SQLite journal remains after refresh timeout: ${journal_path}. Skipping cache staging for this partial refresh."
stage_cache=false
git restore --worktree -- "${db_path}" 2>/dev/null || true
rm -f "${journal_path}"
return
fi
python3 -c 'import sqlite3, sys; db_path = sys.argv[1]; connection = sqlite3.connect(db_path); result = connection.execute("PRAGMA integrity_check").fetchone(); connection.close(); raise SystemExit(0 if result and result[0] == "ok" else f"SQLite integrity check failed for {db_path}: {result}")' "${db_path}"
if [[ -e "${journal_path}" ]]; then
echo "::error::SQLite journal remains after recovery attempt: ${journal_path}. Refusing to stage a potentially inconsistent cache."
exit 1
fi
}
recover_sqlite_database "${site_dir}/data/cache/places.sqlite"
cleanup_runtime_artifacts "${site_dir}/data/raw"
cleanup_runtime_artifacts "${site_dir}/data/cache"
cleanup_runtime_artifacts "${site_dir}/public/place-photos"
cleanup_runtime_artifacts "${site_dir}/public/author-photos"
remaining_journal="$(
find "${site_dir}/data/cache" \( -name "*-journal" -o -name "*.sqlite-journal" \) -print -quit 2>/dev/null || true
)"
if [[ -n "${remaining_journal}" ]]; then
if [[ "${REFRESH_TIMED_OUT}" == "true" ]]; then
echo "::warning::SQLite journal remains after refresh timeout: ${remaining_journal}. Skipping cache staging for this partial refresh."
stage_cache=false
git restore --worktree -- "${site_dir}/data/cache" 2>/dev/null || true
find "${site_dir}/data/cache" \( -name "*-journal" -o -name "*.sqlite-journal" \) -delete 2>/dev/null || true
else
echo "::error::SQLite journal remains after refresh interruption: ${remaining_journal}. Refusing to stage a potentially inconsistent cache."
exit 1
fi
fi
git add -A -- "${site_dir}/data/raw"
if [[ "${stage_cache}" == "true" ]]; then
git add -A -- "${site_dir}/data/cache"
fi
stage_optional_path "${site_dir}/public/place-photos"
stage_optional_path "${site_dir}/public/author-photos"
if git diff --cached --quiet; then
echo "changed=false" >> "${GITHUB_OUTPUT}"
exit 0
fi
commit_subject="Refresh place data (${PROFILE})"
if [[ "${REFRESH_TIMED_OUT}" == "true" ]]; then
commit_subject="${commit_subject} [partial]"
fi
git commit -m "${commit_subject}"
git push --force-with-lease origin "${REFRESH_BRANCH}"
echo "changed=true" >> "${GITHUB_OUTPUT}"
- name: Create or update pull request
if: steps.commit.outputs.changed == 'true'
uses: actions/github-script@v8
env:
PROFILE: ${{ steps.settings.outputs.profile }}
REFRESH_BRANCH: ${{ steps.settings.outputs.refresh_branch }}
REFRESH_DATE: ${{ steps.settings.outputs.refresh_date }}
REFRESH_LIST: ${{ steps.settings.outputs.refresh_list }}
REFRESH_FORCE: ${{ steps.settings.outputs.refresh_force }}
REFRESH_SUMMARY_PATH: ${{ runner.temp }}/data-refresh-summary.md
SKIP_PHOTOS: ${{ steps.settings.outputs.skip_photos }}
REFRESH_WORKERS: ${{ steps.settings.outputs.refresh_workers }}
SOFT_TIMEOUT_MINUTES: ${{ steps.settings.outputs.soft_timeout_minutes }}
REFRESH_TIMED_OUT: ${{ steps.refresh.outputs.timed_out }}
with:
github-token: ${{ secrets.GH_AUTOMATION_TOKEN != '' && secrets.GH_AUTOMATION_TOKEN || github.token }}
script: |
const fs = require("fs");
const owner = context.repo.owner;
const repo = context.repo.repo;
const head = `${owner}:${process.env.REFRESH_BRANCH}`;
const title = `Refresh place data (${process.env.PROFILE}, ${process.env.REFRESH_DATE})`;
const manualRun = context.eventName === "workflow_dispatch";
const bodyLines = [
"Automated self-hosted data refresh.",
"",
`- Refresh date: \`${process.env.REFRESH_DATE}\``,
`- Profile: \`${process.env.PROFILE}\``,
`- Trigger: \`${context.eventName}\``,
`- Refresh workers: \`${process.env.REFRESH_WORKERS}\``,
`- Soft timeout: \`${process.env.SOFT_TIMEOUT_MINUTES} minutes\``,
];
if (process.env.REFRESH_LIST) {
bodyLines.push(`- Refresh list: \`${process.env.REFRESH_LIST}\``);
}
if (process.env.REFRESH_FORCE === "true") {
bodyLines.push("- Raw source refresh forced");
}
if (process.env.SKIP_PHOTOS === "true") {
bodyLines.push("- Photo refresh skipped");
}
if (process.env.REFRESH_TIMED_OUT === "true") {
bodyLines.push("- Partial refresh: stopped at the soft timeout so completed changes could be committed");
}
if (manualRun) {
bodyLines.push(`- Manual run: https://github.com/${owner}/${repo}/actions/runs/${context.runId}`);
}
if (process.env.REFRESH_SUMMARY_PATH && fs.existsSync(process.env.REFRESH_SUMMARY_PATH)) {
const summary = fs.readFileSync(process.env.REFRESH_SUMMARY_PATH, "utf8").trim();
if (summary) {
bodyLines.push("", "## Summary", "", summary);
}
}
const body = bodyLines.join("\n");
const { data: pulls } = await github.rest.pulls.list({
owner,
repo,
state: "open",
head,
base: "main",
});
if (pulls.length > 0) {
await github.rest.pulls.update({
owner,
repo,
pull_number: pulls[0].number,
title,
body,
});
core.info(`Updated PR #${pulls[0].number}`);
return;
}
const { data: pull } = await github.rest.pulls.create({
owner,
repo,
title,
head: process.env.REFRESH_BRANCH,
base: "main",
body,
});
core.info(`Created PR #${pull.number}`);
- name: Close stale pull request
if: steps.commit.outputs.changed != 'true'
uses: actions/github-script@v8
env:
REFRESH_BRANCH: ${{ steps.settings.outputs.refresh_branch }}
with:
github-token: ${{ secrets.GH_AUTOMATION_TOKEN != '' && secrets.GH_AUTOMATION_TOKEN || github.token }}
script: |
const owner = context.repo.owner;
const repo = context.repo.repo;
const head = `${owner}:${process.env.REFRESH_BRANCH}`;
const { data: pulls } = await github.rest.pulls.list({
owner,
repo,
state: "open",
head,
base: "main",
});
if (pulls.length === 0) {
core.info(`No open PR found for ${head}`);
return;
}
await github.rest.pulls.update({
owner,
repo,
pull_number: pulls[0].number,
state: "closed",
});
core.info(`Closed PR #${pulls[0].number}`);
- name: Delete stale remote branch
if: steps.commit.outputs.changed != 'true'
shell: bash
env:
REFRESH_BRANCH: ${{ steps.settings.outputs.refresh_branch }}
run: |
set -euo pipefail
if git ls-remote --exit-code --heads origin "${REFRESH_BRANCH}" >/dev/null 2>&1; then
git push origin --delete "${REFRESH_BRANCH}"
echo "Deleted remote branch ${REFRESH_BRANCH}."
else
echo "No remote branch to delete for ${REFRESH_BRANCH}."
fi
- name: Summarize result
shell: bash
env:
REFRESH_BRANCH: ${{ steps.settings.outputs.refresh_branch }}
run: |
set -euo pipefail
if [[ "${{ steps.commit.outputs.changed }}" == "true" ]]; then
echo "Opened or updated a PR from ${REFRESH_BRANCH}."
else
echo "No tracked data changes detected; closed any matching PR and removed ${REFRESH_BRANCH}."
fi