Enumerate failing specs/E2E from CI artifacts; mute all github-actions comments

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DgBEW6qAwgn2fcQbA2f4ZQ

server/config.ts

@@ -6,6 +6,18 @@ import type { CheckCategory } from '../shared/types.js';
 
 const categorySchema = z.array(z.string());
 
+const failureReportSchema = z.object({
+  /** Check-run name this applies to; supports `*` globs. */
+  check: z.string(),
+  /** Artifact name on the workflow run; `{number}` = PR number, supports `*` globs. */
+  artifact: z.string(),
+  parser: z.enum(['rspec', 'playwright']),
+  /** Grouping label shown in the UI (e.g. "specs", "e2e"). */
+  label: z.string(),
+});
+
+export type FailureReportRule = z.infer<typeof failureReportSchema>;
+
 const repoSchema = z.object({
   name: z.string().regex(/^[\w.-]+\/[\w.-]+$/),
   checkCategories: z
@@ -16,6 +28,7 @@ const repoSchema = z.object({
     })
     .default({}),
   ignoreChecks: z.array(z.string()).default([]),
+  failureReports: z.array(failureReportSchema).default([]),
   preview: z
     .object({
       label: z.string(),
@@ -52,11 +65,21 @@ interface GlobMatcher {
   category: CheckCategory;
 }
 
+export interface CompiledFailureRule {
+  matchesCheck: (name: string) => boolean;
+  artifact: string;
+  parser: 'rspec' | 'playwright';
+  label: string;
+  /** Stable identity for caching: the raw check pattern. */
+  key: string;
+}
+
 export interface RepoRules {
   exact: Map<string, CheckCategory>;
   globs: GlobMatcher[];
   ignore: Set<string>;
   preview?: { label: string; urlTemplate: string };
+  failureRules: CompiledFailureRule[];
 }
 
 export interface AppConfig extends RawConfig {
@@ -113,6 +136,13 @@ export function loadConfig(path = join(rootDir, 'config.json')): AppConfig {
       globs,
       ignore: new Set(repo.ignoreChecks),
       preview: repo.preview,
+      failureRules: repo.failureReports.map((r) => ({
+        matchesCheck: GLOB_CHARS.test(r.check) ? compileNameGlob(r.check) : (n) => n === r.check,
+        artifact: r.artifact,
+        parser: r.parser,
+        label: r.label,
+        key: r.check,
+      })),
     });
   }
 

server/db.ts

@@ -38,11 +38,32 @@ CREATE TABLE IF NOT EXISTS events (
   read_at      TEXT
 );
 
+CREATE TABLE IF NOT EXISTS failure_reports (
+  pr_id      TEXT NOT NULL,
+  key        TEXT NOT NULL,
+  label      TEXT NOT NULL,
+  run_id     TEXT NOT NULL,
+  fetched_at TEXT NOT NULL,
+  failed     INTEGER NOT NULL,
+  truncated  INTEGER NOT NULL DEFAULT 0,
+  failures   TEXT NOT NULL,
+  PRIMARY KEY (pr_id, key)
+);
+
 CREATE INDEX IF NOT EXISTS idx_events_created ON events (created_at DESC);
 CREATE INDEX IF NOT EXISTS idx_events_pr      ON events (pr_id);
 CREATE INDEX IF NOT EXISTS idx_events_unread  ON events (read) WHERE read = 0;
 `;
 
+export interface FailureReportRow {
+  key: string;
+  label: string;
+  fetched_at: string;
+  failed: number;
+  truncated: number;
+  failures: string;
+}
+
 export interface EventInsert {
   id: string;
   prId: string;
@@ -184,6 +205,77 @@ export class Db {
       );
   }
 
+  getFailureReportRunId(prId: string, key: string): string | null {
+    const row = this.db
+      .prepare('SELECT run_id FROM failure_reports WHERE pr_id = ? AND key = ?')
+      .get(prId, key) as { run_id: string } | undefined;
+    return row?.run_id ?? null;
+  }
+
+  upsertFailureReport(
+    prId: string,
+    key: string,
+    label: string,
+    runId: string,
+    report: { failed: number; truncated: boolean; failures: unknown[] },
+  ): void {
+    this.db
+      .prepare(
+        `INSERT INTO failure_reports (pr_id, key, label, run_id, fetched_at, failed, truncated, failures)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(pr_id, key) DO UPDATE SET
+           label = excluded.label, run_id = excluded.run_id, fetched_at = excluded.fetched_at,
+           failed = excluded.failed, truncated = excluded.truncated, failures = excluded.failures`,
+      )
+      .run(
+        prId,
+        key,
+        label,
+        runId,
+        new Date().toISOString(),
+        report.failed,
+        report.truncated ? 1 : 0,
+        JSON.stringify(report.failures),
+      );
+  }
+
+  /** Remove reports whose rule no longer has failing checks. Returns true if any removed. */
+  pruneFailureReports(prId: string, activeKeys: string[]): boolean {
+    const placeholders = activeKeys.map(() => '?').join(', ');
+    const sql = activeKeys.length
+      ? `DELETE FROM failure_reports WHERE pr_id = ? AND key NOT IN (${placeholders})`
+      : 'DELETE FROM failure_reports WHERE pr_id = ?';
+    return Number(this.db.prepare(sql).run(prId, ...activeKeys).changes) > 0;
+  }
+
+  failureReportRows(prId: string): FailureReportRow[] {
+    return this.db
+      .prepare(
+        'SELECT key, label, fetched_at, failed, truncated, failures FROM failure_reports WHERE pr_id = ? ORDER BY label',
+      )
+      .all(prId) as unknown as FailureReportRow[];
+  }
+
+  failureSummaries(): Record<string, { key: string; label: string; failed: number; truncated: boolean }[]> {
+    const rows = this.db
+      .prepare(
+        `SELECT f.pr_id, f.key, f.label, f.failed, f.truncated
+         FROM failure_reports f JOIN prs p ON p.id = f.pr_id AND p.state = 'OPEN'
+         ORDER BY f.label`,
+      )
+      .all() as { pr_id: string; key: string; label: string; failed: number; truncated: number }[];
+    const out: Record<string, { key: string; label: string; failed: number; truncated: boolean }[]> = {};
+    for (const r of rows) {
+      (out[r.pr_id] ??= []).push({
+        key: r.key,
+        label: r.label,
+        failed: r.failed,
+        truncated: r.truncated === 1,
+      });
+    }
+    return out;
+  }
+
   getSnapshot(id: string): PrSnapshot | null {
     const row = this.db.prepare('SELECT snapshot FROM prs WHERE id = ?').get(id) as
       | { snapshot: string }

server/failures.ts

@@ -0,0 +1,192 @@
+import { unzipSync, strFromU8 } from 'fflate';
+import type { FailureReport, PrSnapshot, SpecFailure } from '../shared/types.js';
+import type { CompiledFailureRule, RepoRules } from './config.js';
+import type { Db } from './db.js';
+import { restJson, restZip } from './github/rest.js';
+
+const MAX_ARTIFACT_BYTES = 80 * 1024 * 1024;
+const MAX_STORED_FAILURES = 1000;
+const MSG_LEN = 240;
+
+interface ArtifactInfo {
+  id: number;
+  name: string;
+  expired: boolean;
+  size_in_bytes: number;
+}
+
+function firstLine(text: string): string {
+  const line = text.split('\n').find((l) => l.trim() !== '') ?? '';
+  return line.trim().slice(0, MSG_LEN);
+}
+
+/* ---------- parsers ---------- */
+
+interface RspecJson {
+  examples?: {
+    status: string;
+    file_path: string;
+    line_number: number;
+    full_description: string;
+    exception?: { message?: string };
+  }[];
+}
+
+function parseRspecZip(zip: Uint8Array): SpecFailure[] {
+  const files = unzipSync(zip, { filter: (f) => f.name.endsWith('.json') });
+  const failures: SpecFailure[] = [];
+  for (const [, data] of Object.entries(files)) {
+    let parsed: RspecJson;
+    try {
+      parsed = JSON.parse(strFromU8(data)) as RspecJson;
+    } catch {
+      continue;
+    }
+    for (const ex of parsed.examples ?? []) {
+      if (ex.status !== 'failed') continue;
+      failures.push({
+        path: ex.file_path.replace(/^\.\//, ''),
+        line: ex.line_number ?? null,
+        name: ex.full_description,
+        message: firstLine(ex.exception?.message ?? ''),
+      });
+    }
+  }
+  return failures;
+}
+
+/** Playwright HTML-report artifacts carry one `data/<hash>.md` per failed test
+ *  (the "copy prompt" attachment) with Name / Location / Error details blocks. */
+function parsePlaywrightZip(zip: Uint8Array): SpecFailure[] {
+  const files = unzipSync(zip, {
+    filter: (f) => f.name.startsWith('data/') && f.name.endsWith('.md'),
+  });
+  const failures: SpecFailure[] = [];
+  const seen = new Set<string>();
+  for (const [, data] of Object.entries(files)) {
+    const text = strFromU8(data);
+    const name = /^- Name: (.+)$/m.exec(text)?.[1]?.trim();
+    const location = /^- Location: (.+)$/m.exec(text)?.[1]?.trim();
+    if (!name || seen.has(name + location)) continue;
+    seen.add(name + location);
+    const errorBlock = /# Error details\s*```([\s\S]*?)```/.exec(text)?.[1] ?? '';
+    const locMatch = location ? /^(.*?):(\d+)(?::\d+)?$/.exec(location) : null;
+    failures.push({
+      path: locMatch?.[1] ?? location ?? '',
+      line: locMatch?.[2] ? Number(locMatch[2]) : null,
+      name: name.replace(/ >> /g, ' › '),
+      message: firstLine(errorBlock),
+    });
+  }
+  return failures;
+}
+
+/* ---------- orchestration ---------- */
+
+function runIdFromUrl(url: string | null): string | null {
+  if (!url) return null;
+  return /\/actions\/runs\/(\d+)\//.exec(url)?.[1] ?? null;
+}
+
+function artifactMatcher(pattern: string, prNumber: number): (name: string) => boolean {
+  const concrete = pattern.replaceAll('{number}', String(prNumber));
+  if (!concrete.includes('*')) return (n) => n === concrete;
+  const regex = new RegExp(
+    `^${concrete.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*')}$`,
+  );
+  return (n) => regex.test(n);
+}
+
+/** Sync failure reports for one PR: for each configured rule with at least one
+ *  failing check, download + parse that run's artifacts unless the stored report
+ *  already covers the same run. Reports for now-passing rules are deleted. */
+export async function syncFailureReports(
+  db: Db,
+  rules: RepoRules | undefined,
+  repo: string,
+  snapshot: PrSnapshot,
+): Promise<boolean> {
+  if (!rules || rules.failureRules.length === 0) return false;
+
+  const failingChecks = snapshot.categories
+    .flatMap((c) => c.checks)
+    .filter((c) => c.state === 'fail');
+
+  const activeKeys: string[] = [];
+  let changed = false;
+
+  for (const rule of rules.failureRules) {
+    const matches = failingChecks.filter((c) => rule.matchesCheck(c.name));
+    if (matches.length === 0) continue;
+    activeKeys.push(rule.key);
+
+    const runId = matches.map((m) => runIdFromUrl(m.url)).find((r) => r !== null);
+    if (!runId) continue;
+    const existing = db.getFailureReportRunId(snapshot.id, rule.key);
+    if (existing === runId) continue;
+
+    try {
+      const failures = await fetchAndParse(repo, runId, rule, snapshot.number);
+      db.upsertFailureReport(snapshot.id, rule.key, rule.label, runId, {
+        failed: failures.length,
+        truncated: failures.length > MAX_STORED_FAILURES,
+        failures: failures.slice(0, MAX_STORED_FAILURES),
+      });
+      changed = true;
+      console.log(
+        `[failures] ${repo}#${snapshot.number} ${rule.label}: ${failures.length} failing`,
+      );
+    } catch (err) {
+      console.error(
+        `[failures] ${repo}#${snapshot.number} ${rule.key}: ${err instanceof Error ? err.message : err}`,
+      );
+    }
+  }
+
+  changed = db.pruneFailureReports(snapshot.id, activeKeys) || changed;
+  return changed;
+}
+
+async function fetchAndParse(
+  repo: string,
+  runId: string,
+  rule: CompiledFailureRule,
+  prNumber: number,
+): Promise<SpecFailure[]> {
+  const { artifacts } = await restJson<{ artifacts: ArtifactInfo[] }>(
+    `/repos/${repo}/actions/runs/${runId}/artifacts?per_page=100`,
+  );
+  const match = artifactMatcher(rule.artifact, prNumber);
+  const wanted = artifacts.filter((a) => match(a.name) && !a.expired);
+
+  const failures: SpecFailure[] = [];
+  for (const artifact of wanted) {
+    if (artifact.size_in_bytes > MAX_ARTIFACT_BYTES) {
+      console.warn(`[failures] skipping oversized artifact ${artifact.name}`);
+      continue;
+    }
+    const zip = await restZip(`/repos/${repo}/actions/artifacts/${artifact.id}/zip`, MAX_ARTIFACT_BYTES);
+    if (!zip) continue;
+    failures.push(...(rule.parser === 'rspec' ? parseRspecZip(zip) : parsePlaywrightZip(zip)));
+  }
+  failures.sort((a, b) => a.path.localeCompare(b.path) || (a.line ?? 0) - (b.line ?? 0));
+  return failures;
+}
+
+export function buildFailureReport(row: {
+  key: string;
+  label: string;
+  fetched_at: string;
+  failed: number;
+  truncated: number;
+  failures: string;
+}): FailureReport {
+  return {
+    key: row.key,
+    label: row.label,
+    fetchedAt: row.fetched_at,
+    failed: row.failed,
+    truncated: row.truncated === 1,
+    failures: JSON.parse(row.failures) as SpecFailure[],
+  };
+}

server/github/rest.ts

@@ -0,0 +1,29 @@
+import { getToken } from './token.js';
+import { GithubError } from './client.js';
+
+const BASE = 'https://api.github.com';
+
+export async function restJson<T>(path: string): Promise<T> {
+  const res = await fetch(`${BASE}${path}`, {
+    headers: {
+      authorization: `bearer ${getToken()}`,
+      accept: 'application/vnd.github+json',
+      'user-agent': 'pr-monitor',
+    },
+  });
+  if (!res.ok) {
+    throw new GithubError(`GitHub REST ${path}: ${res.status}`);
+  }
+  return res.json() as Promise<T>;
+}
+
+/** Download an artifact zip (follows the storage redirect). */
+export async function restZip(path: string, maxBytes: number): Promise<Uint8Array | null> {
+  const res = await fetch(`${BASE}${path}`, {
+    headers: { authorization: `bearer ${getToken()}`, 'user-agent': 'pr-monitor' },
+  });
+  if (!res.ok) throw new GithubError(`GitHub REST ${path}: ${res.status}`);
+  const buf = new Uint8Array(await res.arrayBuffer());
+  if (buf.byteLength > maxBytes) return null;
+  return buf;
+}

server/poller.ts

@@ -126,6 +126,12 @@ export async function runPoll(config: AppConfig, db: Db): Promise<PollResult> {
     db.upsertPr(snapshot, signature(pr), detail ? now : null);
     if (detail) {
       newEventCount += ingest(db, detail, config.user, backfillAsRead);
+      await syncFailureReports(
+        db,
+        config.rulesByRepo.get(snapshot.repo),
+        snapshot.repo,
+        snapshot,
+      );
     }
   }
 
@@ -137,6 +143,7 @@ export async function runPoll(config: AppConfig, db: Db): Promise<PollResult> {
 
 // Thin wrapper so poller stays the single import for index.ts / sync-once.
 import { ingestTimeline } from './timeline.js';
+import { syncFailureReports } from './failures.js';
 function ingest(db: Db, detail: DetailPr, user: string, backfillAsRead: boolean): number {
   return ingestTimeline(db, detail, { user, backfillAsRead });
 }

server/routes.ts

@@ -5,6 +5,7 @@ import type { StatePayload, SyncEventPayload } from '../shared/types.js';
 import type { Db } from './db.js';
 import { rootDir } from './config.js';
 import { addClient } from './sse.js';
+import { buildFailureReport } from './failures.js';
 
 export interface RouteDeps {
   db: Db;
@@ -33,10 +34,15 @@ export function createApp({ db, triggerSync }: RouteDeps): Express {
       lastSyncError: db.getMeta('last_sync_error'),
       unreadCountsByPr: byPr,
       unreadTotal: total,
+      failuresByPr: db.failureSummaries(),
     };
     res.json(payload);
   });
 
+  app.get('/api/prs/:id/failures', (req, res) => {
+    res.json(db.failureReportRows(req.params.id).map(buildFailureReport));
+  });
+
   app.get('/api/timeline', (req, res) => {
     const limit = Math.min(Number(req.query.limit) || 50, 200);
     const before = typeof req.query.before === 'string' ? req.query.before : undefined;