apps/web/src/routes/api/cron/pinecone-refresh.ts

1import { timingSafeEqual } from 'node:crypto'
2import {
3	upsertFaqCatalogue,
4	upsertMediaCatalogue,
5	upsertSitemapCatalogue,
6	upsertTextContent,
7} from '@local/ai/components/api/pineconeUpload'
8import type { APIEvent } from '@solidjs/start/server'
9import { fetchAgendaForScope } from '~/lib/agenda'
10import { faqCatalogueRecords, fetchFaqsFromSanity } from '~/lib/faqCatalogue'
11import {
12	fetchMediaFromSanity,
13	formatMediaItemText,
14	mediaItemId,
15} from '~/lib/media'
16import {
17	eventPageContentRecords,
18	eventPageMetaRecord,
19	fetchEventPagesWithContent,
20	fetchSitePagesFromSanity,
21	formatSitePageText,
22	isEventSitePage,
23	pageVectorMetadata,
24	sitePageId,
25} from '~/lib/pageCatalogue'
26
27/**
28 * Nightly full refresh of every Sanity/Inwink-derived vector store
29 * (scheduled in apps/web/vercel.json). The publish webhook keeps stores
30 * fresh incrementally; this is the safety net that also catches builder
31 * changes (new metadata, new chunking) which a webhook never replays.
32 *
33 * Vercel invokes cron paths with GET and `Authorization: Bearer $CRON_SECRET`.
34 * INTERNAL_API_SECRET is accepted too so the run can be triggered by hand.
35 */
36
37const LOG_PREFIX = '[Pinecone cron]'
38
39function bearerMatches(header: string, secret: string | undefined): boolean {
40	const value = secret?.trim()
41	if (!value) return false
42	const expected = Buffer.from(`Bearer ${value}`)
43	const received = Buffer.from(header)
44	return (
45		expected.length === received.length && timingSafeEqual(expected, received)
46	)
47}
48
49function isAuthorized(request: Request): boolean {
50	const header = request.headers.get('authorization') ?? ''
51	return (
52		bearerMatches(header, process.env.CRON_SECRET) ||
53		bearerMatches(header, process.env.INTERNAL_API_SECRET)
54	)
55}
56
57type StepResult =
58	| { ok: true; totalVectors: number; staleDeleted?: number }
59	| { ok: false; error: string }
60
61async function runStep(
62	name: string,
63	fn: () => Promise<{ totalVectors: number; staleDeleted?: number }>,
64): Promise<[string, StepResult]> {
65	try {
66		const result = await fn()
67		return [name, { ok: true, ...result }]
68	} catch (error) {
69		const message = error instanceof Error ? error.message : 'failed'
70		console.error(`${LOG_PREFIX} ${name} failed`, error)
71		return [name, { ok: false, error: message }]
72	}
73}
74
75async function refreshFaqs() {
76	return upsertFaqCatalogue(faqCatalogueRecords(await fetchFaqsFromSanity()))
77}
78
79async function refreshMedia() {
80	const items = await fetchMediaFromSanity()
81	return upsertMediaCatalogue(
82		items.map((item) => ({
83			id: mediaItemId(item.type, item.slug),
84			text: formatMediaItemText(item),
85			metadata: { type: item.type, slug: item.slug, title: item.title },
86		})),
87	)
88}
89
90async function refreshSitemap() {
91	const pages = await fetchSitePagesFromSanity()
92	const eventPages = await fetchEventPagesWithContent()
93	return upsertSitemapCatalogue([
94		...pages
95			.filter((page) => !isEventSitePage(page))
96			.map((page) => ({
97				id: sitePageId(page),
98				text: formatSitePageText(page),
99				metadata: pageVectorMetadata(page),
100			})),
101		...eventPages.map(eventPageMetaRecord),
102		...eventPages.flatMap(eventPageContentRecords),
103	])
104}
105
106async function refreshAgenda(scopeKey: 'event-world' | 'event-america') {
107	const { markdown } = await fetchAgendaForScope(scopeKey)
108	return upsertTextContent(scopeKey, markdown)
109}
110
111export async function GET({ request }: APIEvent) {
112	if (!isAuthorized(request)) {
113		return Response.json({ ok: false, error: 'Unauthorized' }, { status: 401 })
114	}
115	const startedAt = Date.now()
116	// Sequential on purpose: the steps share the OpenAI embedding quota and
117	// one failure must not abort the others.
118	const steps = Object.fromEntries([
119		await runStep('faq', refreshFaqs),
120		await runStep('media', refreshMedia),
121		await runStep('sitemap', refreshSitemap),
122		await runStep('agenda:event-world', () => refreshAgenda('event-world')),
123		await runStep('agenda:event-america', () => refreshAgenda('event-america')),
124	])
125	const ok = Object.values(steps).every((step) => step.ok)
126	return Response.json(
127		{ ok, durationMs: Date.now() - startedAt, steps },
128		{ status: ok ? 200 : 500 },
129	)
130}
131