|
| 1 | +#!/bin/bash |
| 2 | +# |
| 3 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 4 | +# or more contributor license agreements. See the NOTICE file |
| 5 | +# distributed with this work for additional information |
| 6 | +# regarding copyright ownership. The ASF licenses this file |
| 7 | +# to you under the Apache License, Version 2.0 (the |
| 8 | +# "License"); you may not use this file except in compliance |
| 9 | +# with the License. You may obtain a copy of the License at |
| 10 | +# |
| 11 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | +# |
| 13 | +# Unless required by applicable law or agreed to in writing, |
| 14 | +# software distributed under the License is distributed on an |
| 15 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 16 | +# KIND, either express or implied. See the License for the |
| 17 | +# specific language governing permissions and limitations |
| 18 | +# under the License. |
| 19 | +# |
| 20 | +# Validate the agent-friendly documentation (agentdocsspec.com) — the Markdown mirror, the |
| 21 | +# llms.txt discovery index, page sizes, link resolution, and the per-page directive — using the |
| 22 | +# spec's own tool, `npx afdocs check`. |
| 23 | +# |
| 24 | +# Usage: |
| 25 | +# bin/validate-llms-txt.sh # check the published current docs |
| 26 | +# # (https://tinkerpop.apache.org/docs/current/) |
| 27 | +# bin/validate-llms-txt.sh -v 3.7.7 # check a published version |
| 28 | +# bin/validate-llms-txt.sh --version 3.7.7 # (https://tinkerpop.apache.org/docs/3.7.7/) |
| 29 | +# bin/validate-llms-txt.sh -v local # check the LOCAL build: replicate the publish layout |
| 30 | +# # (markdown + html + images together) under |
| 31 | +# # target/docs/serve/, serve it, and check localhost |
| 32 | +# |
| 33 | +# Any extra arguments after the mode are passed through to `afdocs check` (e.g. --fixes, --verbose, |
| 34 | +# --format json, --skip-checks ...). |
| 35 | +# |
| 36 | +# NOTE: afdocs requires Node.js >= 22; this script warns if the active node is older. A plain local |
| 37 | +# static server cannot exercise the hosting-dependent checks (content-negotiation, redirects, |
| 38 | +# soft-404, cache headers, auth) — those are only meaningful against the real published site. |
| 39 | + |
| 40 | +set -e |
| 41 | + |
| 42 | +cd "$(dirname "$0")/.." |
| 43 | +TP_HOME="$(pwd)" |
| 44 | + |
| 45 | +BASE_URL="https://tinkerpop.apache.org/docs" |
| 46 | +VERSION="current" |
| 47 | + |
| 48 | +# --------------------------------------------------------------------------- |
| 49 | +# Argument parsing |
| 50 | +# --------------------------------------------------------------------------- |
| 51 | +PASSTHROUGH=() |
| 52 | +while [[ $# -gt 0 ]]; do |
| 53 | + case "$1" in |
| 54 | + -v|--version) VERSION="$2"; shift 2 ;; |
| 55 | + -h|--help) |
| 56 | + sed -n '19,40p' "$0" | sed 's/^# \{0,1\}//' |
| 57 | + exit 0 ;; |
| 58 | + *) PASSTHROUGH+=("$1"); shift ;; |
| 59 | + esac |
| 60 | +done |
| 61 | + |
| 62 | +# --------------------------------------------------------------------------- |
| 63 | +# Node version advisory (afdocs requires >= 22) |
| 64 | +# --------------------------------------------------------------------------- |
| 65 | +if command -v node >/dev/null 2>&1; then |
| 66 | + NODE_MAJOR=$(node -v | sed 's/^v//; s/\..*//') |
| 67 | + if [ "${NODE_MAJOR:-0}" -lt 22 ]; then |
| 68 | + echo "WARNING: afdocs requires Node.js >= 22 but found $(node -v)." |
| 69 | + echo " Results may be unreliable. Install a newer node (e.g. 'mise install node@22')." |
| 70 | + fi |
| 71 | +else |
| 72 | + echo "ERROR: node/npx not found on PATH; afdocs needs Node.js >= 22." |
| 73 | + exit 1 |
| 74 | +fi |
| 75 | + |
| 76 | +# --------------------------------------------------------------------------- |
| 77 | +# Local mode: replicate the publish layout and serve it |
| 78 | +# --------------------------------------------------------------------------- |
| 79 | +HTTP_PID="" |
| 80 | +cleanup() { |
| 81 | + set +e |
| 82 | + [ -n "${HTTP_PID}" ] && kill "${HTTP_PID}" 2>/dev/null |
| 83 | +} |
| 84 | +trap cleanup EXIT INT TERM |
| 85 | + |
| 86 | +run_afdocs() { |
| 87 | + local url="$1"; shift |
| 88 | + local extra=("$@") |
| 89 | + echo "Running: npx --yes afdocs check ${url} ${extra[*]} ${PASSTHROUGH[*]}" |
| 90 | + echo "-----------------------------------------------------------------------" |
| 91 | + npx --yes afdocs@latest check "${url}" "${extra[@]}" "${PASSTHROUGH[@]}" |
| 92 | +} |
| 93 | + |
| 94 | +if [ "${VERSION}" == "local" ]; then |
| 95 | + MD_DIR="target/docs/markdown" |
| 96 | + HTML_DIR="target/docs/htmlsingle" |
| 97 | + SERVE_DIR="target/docs/serve" |
| 98 | + |
| 99 | + if [ ! -d "${MD_DIR}" ] || [ ! -f "${MD_DIR}/llms.txt" ]; then |
| 100 | + echo "ERROR: No local Markdown docs found at ${MD_DIR} (with llms.txt)." |
| 101 | + echo " Generate them first, e.g.: bin/process-docs.sh --dryRun" |
| 102 | + echo " (a --dryRun build produces the Markdown mirror + llms.txt without a live server;" |
| 103 | + echo " a full 'bin/process-docs.sh' additionally fills in executed ==> output.)" |
| 104 | + exit 1 |
| 105 | + fi |
| 106 | + |
| 107 | + echo "Replicating publish layout into ${SERVE_DIR}/ (markdown + html + images together)..." |
| 108 | + rm -rf "${SERVE_DIR}" |
| 109 | + mkdir -p "${SERVE_DIR}" |
| 110 | + # HTML first (brings images/, stylesheets, and the .html pages), then overlay the Markdown mirror |
| 111 | + # so .md sits beside .html and llms.txt lands at the served root — mirroring bin/publish-docs.sh. |
| 112 | + if [ -d "${HTML_DIR}" ]; then |
| 113 | + cp -R "${HTML_DIR}/." "${SERVE_DIR}/" |
| 114 | + else |
| 115 | + echo "NOTE: ${HTML_DIR} not found; serving Markdown only (HTML/parity checks will be limited)." |
| 116 | + fi |
| 117 | + cp -R "${MD_DIR}/." "${SERVE_DIR}/" |
| 118 | + |
| 119 | + # Pick a free port up front so we can bake the absolute base URL into llms.txt before serving. |
| 120 | + PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()') |
| 121 | + LOCAL_URL="http://127.0.0.1:${PORT}/" |
| 122 | + |
| 123 | + # Regenerate the served llms.txt with ABSOLUTE URLs (http://127.0.0.1:PORT/...). The spec's |
| 124 | + # link-resolution/coverage checks only recognize full http(s):// links, so this mirrors what |
| 125 | + # publish-docs.sh does with the canonical site URL and lets the local run exercise those checks. |
| 126 | + EXT_CLASSES="docs/tinkeradoc-extension/target/classes" |
| 127 | + if [ -d "${EXT_CLASSES}" ]; then |
| 128 | + echo "Regenerating served llms.txt with absolute URLs for ${LOCAL_URL} ..." |
| 129 | + java -cp "${EXT_CLASSES}" org.apache.tinkerpop.tinkeradoc.LlmsTxtGenerator \ |
| 130 | + --prefix "${LOCAL_URL}" --out "${SERVE_DIR}/llms.txt" "${SERVE_DIR}" |
| 131 | + else |
| 132 | + echo "NOTE: ${EXT_CLASSES} not found; serving llms.txt as-is (relative links won't satisfy" |
| 133 | + echo " the link-resolution/coverage checks). Build the extension to enable absolute URLs." |
| 134 | + fi |
| 135 | + |
| 136 | + echo "Serving ${SERVE_DIR}/ at ${LOCAL_URL} ..." |
| 137 | + # A threaded server: afdocs issues concurrent requests, and python's default single-threaded |
| 138 | + # http.server can drop connections mid-response (undici in afdocs then aborts). ThreadingHTTPServer |
| 139 | + # handles the concurrency cleanly. |
| 140 | + python3 -c " |
| 141 | +import sys, http.server, socketserver, functools |
| 142 | +port = int(sys.argv[1]); root = sys.argv[2] |
| 143 | +handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=root) |
| 144 | +class T(socketserver.ThreadingMixIn, http.server.HTTPServer): |
| 145 | + daemon_threads = True |
| 146 | + allow_reuse_address = True |
| 147 | +T(('127.0.0.1', port), handler).serve_forever() |
| 148 | +" "${PORT}" "${SERVE_DIR}" >/dev/null 2>&1 & |
| 149 | + HTTP_PID=$! |
| 150 | + |
| 151 | + # Wait for readiness (loopback works within this single script invocation). |
| 152 | + for i in $(seq 1 30); do |
| 153 | + if curl -sf "http://127.0.0.1:${PORT}/llms.txt" >/dev/null 2>&1; then break; fi |
| 154 | + if ! kill -0 "${HTTP_PID}" 2>/dev/null; then |
| 155 | + echo "ERROR: local web server exited during startup."; exit 1 |
| 156 | + fi |
| 157 | + [ "$i" -eq 30 ] && { echo "ERROR: local web server did not become ready."; exit 1; } |
| 158 | + sleep 0.5 |
| 159 | + done |
| 160 | + |
| 161 | + # Local-run options (each skippable by passing your own): |
| 162 | + # --canonical-origin: tells afdocs the localhost origin is canonical, so the absolute llms.txt |
| 163 | + # links (baked in above) count as same-origin and its link-resolution checks run. |
| 164 | + # --max-concurrency 1 / --request-delay: afdocs' HTTP client (undici) can abort on many-page |
| 165 | + # concurrent crawls against a lightweight local server; serial requests are reliable. |
| 166 | + LOCAL_OPTS=() |
| 167 | + [[ " ${PASSTHROUGH[*]} " == *" --canonical-origin "* ]] || LOCAL_OPTS+=(--canonical-origin "http://127.0.0.1:${PORT}") |
| 168 | + [[ " ${PASSTHROUGH[*]} " == *" --max-concurrency "* ]] || LOCAL_OPTS+=(--max-concurrency 1) |
| 169 | + [[ " ${PASSTHROUGH[*]} " == *" --request-delay "* ]] || LOCAL_OPTS+=(--request-delay 20) |
| 170 | + run_afdocs "${LOCAL_URL}" "${LOCAL_OPTS[@]}" |
| 171 | + echo "-----------------------------------------------------------------------" |
| 172 | + echo "NOTE: hosting-dependent checks (content-negotiation, redirects, soft-404, cache headers," |
| 173 | + echo " auth) are not meaningful against this local static server; validate those against" |
| 174 | + echo " the published site." |
| 175 | +else |
| 176 | + run_afdocs "${BASE_URL}/${VERSION}/" |
| 177 | +fi |
0 commit comments