feat(appkit): metrics describe-cache with retry-driven convergence and last-known-good degradation

Signed-off-by: Atila Fassina <atila@fassina.eu>

Author: atilafassina

packages/appkit/src/type-generator/cache.ts

@@ -2,6 +2,7 @@ import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { createLogger } from "../logging/logger";
+import type { MetricSchema } from "./metric-registry";
 
 const logger = createLogger("type-generator:cache");
 
@@ -16,14 +17,39 @@ interface CacheEntry {
   retry: boolean;
 }
 
+/**
+ * One cached metric-view DESCRIBE outcome.
+ *
+ * `hash` is md5 over `"<source>|<lane>"` — the two config inputs that
+ * determine a DESCRIBE — so editing either invalidates the entry. `schema`
+ * is the full {@link MetricSchema} persisted verbatim (it is JSON-safe by
+ * design), letting a warm pass regenerate both metric artifacts without a
+ * single warehouse call. `retry: true` marks a degraded outcome (DESCRIBE
+ * skipped, unanswered, or failed): the cached schema still renders
+ * artifacts, but the next eligible pass re-describes exactly these keys so
+ * degraded schemas converge to real ones.
+ */
+export interface MetricCacheEntry {
+  hash: string;
+  schema: MetricSchema;
+  retry: boolean;
+}
+
 /**
  * Cache interface
  * @property version - the version of the cache
  * @property queries - the queries in the cache
+ * @property metrics - cached metric-view schemas keyed by metric key.
+ *   OPTIONAL on purpose: version "3" files written before this section
+ *   existed load unchanged (absent ⇒ treated as empty by the metric path),
+ *   and the query path's `noCache` reinit literal stays valid as-is. The
+ *   section rides through the query path's load → mutate → save cycle as a
+ *   plain sibling key, so query-side saves preserve it byte-for-byte.
  */
 interface Cache {
   version: string;
   queries: Record<string, CacheEntry>;
+  metrics?: Record<string, MetricCacheEntry>;
 }
 
 export const CACHE_VERSION = "3";

packages/appkit/src/type-generator/index.ts

@@ -4,6 +4,7 @@ import { WorkspaceClient } from "@databricks/sdk-experimental";
 import dotenv from "dotenv";
 import pc from "picocolors";
 import { createLogger } from "../logging/logger";
+import { hashSQL, loadCache, type MetricCacheEntry, saveCache } from "./cache";
 import {
   createWorkspaceDescribeFetcher,
   type DescribeFetcher,
@@ -256,6 +257,10 @@ async function isWarehouseRunning(
  * @param options - the options for the generation
  * @param options.entryPoint - the entry point file
  * @param options.outFile - the output file
+ * @param options.noCache - skip the typegen cache entirely: every query is
+ *   re-described, and the metric path ignores its cached schemas (every
+ *   configured key becomes describe-needed) and overwrites the cache's
+ *   `metrics` section with this pass's results.
  * @param options.mode - preflight policy (see {@link PreflightMode}). For
  *   queries, `"non-blocking"` never probes or describes the warehouse. For
  *   metric views, `"non-blocking"` makes one status-only probe and DESCRIBEs
@@ -330,12 +335,61 @@ export async function generateFromEntryPoint(options: {
     if (metricConfig) {
       const resolution = resolveMetricConfig(metricConfig);
 
+      // Metric schemas persist in the shared typegen cache as a `metrics`
+      // section (sibling of `queries`, same file, same version) keyed by
+      // metric key with md5("<source>|<lane>") as the change detector. The
+      // cache is (re)loaded here — strictly AFTER generateQueriesFromDescribe
+      // above has finished its own load → mutate → save cycle — so the single
+      // metric-side save below re-serializes the exact `queries` object it
+      // just read and can never clobber a query entry.
+      const cache = await loadCache();
+
+      // The section is consumed through a null-prototype copy: metric keys
+      // are user-controlled config input and "__proto__" passes the metric
+      // key regex — on a plain object, writing it would hit the
+      // Object.prototype setter (mutating the object's prototype and silently
+      // dropping the entry) instead of storing data. A null prototype also
+      // keeps partition reads from resolving inherited names ("constructor",
+      // "toString", ...) as phantom entries.
+      const metricsSection: Record<string, MetricCacheEntry> =
+        Object.create(null);
+      if (!noCache && cache.metrics) {
+        for (const key of Object.keys(cache.metrics)) {
+          metricsSection[key] = cache.metrics[key];
+        }
+      }
+
+      // Partition BEFORE any gate/preflight decision: a hit (hash match and
+      // not flagged for retry) is served from cache no matter what the
+      // warehouse is doing — a degraded-mode pass falls back to
+      // last-known-good schemas exactly like queries degrade to cached
+      // types. Only the remainder — new keys, edited entries, and
+      // retry-flagged degraded entries — is eligible for DESCRIBE, so a
+      // fully-warm pass makes zero warehouse calls and constructs zero
+      // clients. `noCache` left the section empty above, which makes every
+      // configured key describe-needed here.
+      const hitSchemas = new Map<string, MetricSchema>();
+      const describeNeeded: typeof resolution.entries = [];
+      // Parallel to describeNeeded: the config hash to persist per key.
+      const neededHashes: string[] = [];
+      for (const entry of resolution.entries) {
+        const hash = hashSQL(`${entry.source}|${entry.lane}`);
+        const prior = metricsSection[entry.key];
+        if (prior && prior.hash === hash && !prior.retry) {
+          hitSchemas.set(entry.key, prior.schema);
+        } else {
+          describeNeeded.push(entry);
+          neededHashes.push(hash);
+        }
+      }
+
       // At most ONE WorkspaceClient per generation pass for the whole metric
       // path: the non-blocking status probe, the blocking preflight, and the
       // default DESCRIBE fetcher all share this lazily-created instance. A
       // pass that never contacts the warehouse constructs zero clients: an
       // injected metricFetcher covers fetching (and skips probe/preflight),
-      // and an empty metricViews map has nothing to describe in any mode.
+      // and a pass with nothing describe-needed — fully-warm cache or an
+      // empty metricViews map — has nothing to describe in any mode.
       let metricClient: WorkspaceClient | undefined;
       const getMetricClient = (): WorkspaceClient => {
         metricClient ??= new WorkspaceClient({});
@@ -357,7 +411,7 @@ export async function generateFromEntryPoint(options: {
       if (
         mode === "blocking" &&
         metricFetcher === undefined &&
-        resolution.entries.length > 0
+        describeNeeded.length > 0
       ) {
         try {
           const state = await getWarehouseState(getMetricClient(), warehouseId);
@@ -395,40 +449,43 @@ export async function generateFromEntryPoint(options: {
       // start the warehouse) decides whether to describe now or emit degraded
       // artifacts that a later blocking run refreshes. An injected
       // metricFetcher always runs: it doesn't hit a warehouse (tests/CI
-      // inject mocks), so gating it would only skip meaningful work. An empty
-      // metricViews map needs no probe either — nothing would be described
+      // inject mocks), so gating it would only skip meaningful work. A pass
+      // with nothing describe-needed — fully-warm cache or an empty
+      // metricViews map — needs no probe either: nothing would be described
       // in any mode.
       const describeNow =
         metricFetcher !== undefined ||
         mode !== "non-blocking" ||
-        resolution.entries.length === 0 ||
+        describeNeeded.length === 0 ||
         (await isWarehouseRunning(getMetricClient, warehouseId));
 
-      let metricSchemas: MetricSchema[];
+      let described: MetricSchema[];
       let failures: MetricSyncFailure[] = [];
       if (preflightFatalMessage !== undefined) {
         // Fatal preflight (deleted/deleting warehouse): fail exactly like the
         // query path's fatal preflight — skip the DESCRIBE batch, emit
         // degraded schemas so both artifacts are still written, and record
-        // one fatal error per key. The shared end-of-run throw below
+        // one fatal error per describe-needed key (cache hits are unaffected:
+        // they serve their cached schemas). The shared end-of-run throw below
         // (TypegenFatalError, or TypegenSyntaxError's fatalQueries when
         // syntax errors coexist) surfaces them after the writes, identically
         // to query fatals.
-        metricSchemas = resolution.entries.map(emptyMetricSchema);
-        for (const entry of resolution.entries) {
+        described = describeNeeded.map(emptyMetricSchema);
+        for (const entry of describeNeeded) {
           fatalErrors.push({ name: entry.key, message: preflightFatalMessage });
         }
-      } else if (resolution.entries.length === 0) {
-        // Nothing configured to describe: syncMetrics would be a no-op, and
-        // building its default fetcher would construct a client for nothing.
-        // Emit the (empty) artifacts directly.
-        metricSchemas = [];
+      } else if (describeNeeded.length === 0) {
+        // Nothing left to describe — every configured key (if any) was a
+        // cache hit. syncMetrics would be a no-op, and building its default
+        // fetcher would construct a client for nothing. The artifacts below
+        // regenerate from cached schemas alone.
+        described = [];
       } else if (describeNow) {
         const fetcher =
           metricFetcher ??
           createWorkspaceDescribeFetcher(getMetricClient(), warehouseId);
-        ({ schemas: metricSchemas, failures } = await syncMetrics(
-          resolution,
+        ({ schemas: described, failures } = await syncMetrics(
+          { entries: describeNeeded },
           fetcher,
         ));
 
@@ -454,7 +511,7 @@ export async function generateFromEntryPoint(options: {
         // unknown — not errors. One summary line, no per-key warns; failed
         // keys are excluded (the warn loop above already reported them).
         const failedKeys = new Set(failures.map((f) => f.key));
-        const degradedKeys = metricSchemas
+        const degradedKeys = described
           .filter((s) => s.degraded && !failedKeys.has(s.key))
           .map((s) => s.key);
         if (degradedKeys.length > 0) {
@@ -467,20 +524,63 @@ export async function generateFromEntryPoint(options: {
         }
       } else {
         // Deliberately un-probed DESCRIBEs, not failures: emit every
-        // configured key as a degraded schema (permissive types, empty
+        // describe-needed key as a degraded schema (permissive types, empty
         // runtime allowlists) so both artifacts always exist, and say so
-        // once — no per-key warnings (nothing failed). The dev warehouse
-        // watch (or the next blocking run) re-enters this path with the
-        // warehouse RUNNING and lands the real schemas.
-        metricSchemas = resolution.entries.map(emptyMetricSchema);
+        // once — no per-key warnings (nothing failed). Cache hits keep
+        // serving their last-known-good schemas — only the remainder
+        // degrades. The dev warehouse watch (or the next blocking run)
+        // re-enters this path with the warehouse RUNNING and lands the real
+        // schemas.
+        described = describeNeeded.map(emptyMetricSchema);
         logger.info(
           "Warehouse %s is not running — wrote degraded metric types (permissive) for %d metric view(s) (%s); they will refresh once the warehouse is available.",
           warehouseId,
-          resolution.entries.length,
-          resolution.entries.map((e) => e.key).join(", "),
+          describeNeeded.length,
+          describeNeeded.map((e) => e.key).join(", "),
         );
       }
 
+      // Persist this pass's outcomes for exactly the keys it owned (the
+      // describe-needed set): a successful DESCRIBE caches `retry: false`;
+      // every degraded outcome — syncMetrics failures and non-terminal
+      // states, the gate-skip path, and the fatal-preflight path (the last
+      // two never entered syncMetrics) — caches its degraded schema with
+      // `retry: true` so the next eligible pass re-describes only these
+      // keys. Hits were partitioned out above and are never rewritten, which
+      // is what lets a warehouse-down pass keep last-known-good entries
+      // intact. One save per pass; with `noCache` the section was started
+      // empty, so saving overwrites it with this pass's results alone.
+      if (describeNeeded.length > 0 || noCache) {
+        for (let i = 0; i < describeNeeded.length; i++) {
+          // syncMetrics (and both .map(emptyMetricSchema) branches) return
+          // one schema per entry in entry order, so described[i] always
+          // belongs to describeNeeded[i] / neededHashes[i].
+          metricsSection[describeNeeded[i].key] = {
+            hash: neededHashes[i],
+            schema: described[i],
+            retry: described[i].degraded === true,
+          };
+        }
+        cache.metrics = metricsSection;
+        await saveCache(cache);
+      }
+
+      // Merge cached hits with fresh results back into config order
+      // (resolution.entries order — the renderers sort internally where
+      // determinism matters).
+      const describedByKey = new Map<string, MetricSchema>();
+      for (const schema of described) {
+        describedByKey.set(schema.key, schema);
+      }
+      const metricSchemas = resolution.entries.map(
+        (entry) =>
+          hitSchemas.get(entry.key) ??
+          describedByKey.get(entry.key) ??
+          // Unreachable: every entry is either a hit or describe-needed, and
+          // every describe-needed entry yields exactly one schema above.
+          emptyMetricSchema(entry),
+      );
+
       const metricFile =
         metricOutFile ?? path.join(path.dirname(outFile), METRIC_TYPES_FILE);
       const metricDeclarations = generateMetricTypeDeclarations(metricSchemas);