perf(api): use cron triggers to update kv metadata (#935)

* perf(api): simplify kv structure * feat: add all download stats endpoint * perf: add cron trigger to update metadata * Create twenty-parents-marry.md * test: update snapshots
fontsource · Feb 9, 2024 · 8ec7d3e · 8ec7d3e
1 parent c93c388
commit 8ec7d3e
Show file tree

Hide file tree

Showing 22 changed files with 393 additions and 553 deletions.
diff --git a/.changeset/twenty-parents-marry.md b/.changeset/twenty-parents-marry.md
@@ -0,0 +1,5 @@
+---
+"api": patch
+---
+
+perf(api): use cron triggers to update kv metadata
diff --git a/api/metadata/src/fontlist/get.ts b/api/metadata/src/fontlist/get.ts
@@ -1,52 +1,38 @@
-import type { FontsourceMetadata, TTLMetadata } from '../types';
-import type { Fontlist, FontlistQueries } from './types';
+import { KV_TTL, METADATA_KEYS } from '../utils';
+import type { Fontlist, FontlistQueries, MetadataList } from './types';
 import { updateList, updateMetadata } from './update';
 
-const getOrUpdateMetadata = async (
+const getMetadata = async (
 	env: Env,
 	ctx: ExecutionContext,
-): Promise<FontsourceMetadata> => {
-	const { value, metadata } = await env.FONTLIST.getWithMetadata<
-		FontsourceMetadata,
-		TTLMetadata
-	>('metadata', {
+): Promise<MetadataList> => {
+	const value = await env.METADATA.get<MetadataList>(METADATA_KEYS.fonts, {
 		type: 'json',
+		cacheTtl: KV_TTL,
 	});
 
 	if (!value) {
-		return await updateMetadata(env);
-	}
-
-	// If the ttl is not set or the cache expiry is less than the current time, then return old value
-	// while revalidating the cache
-	if (!metadata?.ttl || metadata.ttl < Date.now() / 1000) {
-		ctx.waitUntil(updateMetadata(env));
+		return await updateMetadata(env, ctx);
 	}
 
 	return value;
 };
 
-const getOrUpdateList = async (
+const getList = async (
 	key: FontlistQueries,
 	env: Env,
 	ctx: ExecutionContext,
 ): Promise<Fontlist> => {
-	const { value, metadata } = await env.FONTLIST.getWithMetadata<
-		Fontlist,
-		TTLMetadata
-	>(key, {
+	const value = await env.METADATA.get<Fontlist>(METADATA_KEYS.fontlist(key), {
 		type: 'json',
+		cacheTtl: KV_TTL,
 	});
 
 	if (!value) {
-		return await updateList(key, env);
-	}
-
-	if (!metadata?.ttl || metadata.ttl < Date.now() / 1000) {
-		ctx.waitUntil(updateList(key, env));
+		return await updateList(key, env, ctx);
 	}
 
 	return value;
 };
 
-export { getOrUpdateList, getOrUpdateMetadata };
+export { getList, getMetadata };
diff --git a/api/metadata/src/fontlist/router.ts b/api/metadata/src/fontlist/router.ts
@@ -2,7 +2,7 @@ import { error, type IRequestStrict, json, Router } from 'itty-router';
 
 import type { CFRouterContext } from '../types';
 import { API_BROWSER_TTL, CF_EDGE_TTL } from '../utils';
-import { getOrUpdateList } from './get';
+import { getList } from './get';
 import { type Fontlist, isFontlistQuery } from './types';
 
 const router = Router<IRequestStrict, CFRouterContext>();
@@ -30,7 +30,7 @@ router.get('/fontlist', async (request, env, ctx) => {
 	// If there is no query string, then return the type list
 	let list: Fontlist | undefined;
 	if (queryString.length === 0) {
-		list = await getOrUpdateList('type', env, ctx);
+		list = await getList('type', env, ctx);
 	}
 
 	// If there is a query string, then return the respective list
@@ -44,11 +44,11 @@ router.get('/fontlist', async (request, env, ctx) => {
 		}
 
 		// Get or update the list
-		list = await getOrUpdateList(query, env, ctx);
+		list = await getList(query, env, ctx);
 	}
 
 	if (!list) {
-		return error(500, 'Internal server error.');
+		return error(500, 'Unable to determine query type.');
 	}
 
 	response = json(list, {

diff --git a/api/metadata/src/fontlist/types.ts b/api/metadata/src/fontlist/types.ts
@@ -1,3 +1,5 @@
+import { type IDResponse } from 'common-api/types';
+
 import type { FontMetadata } from '../types';
 
 const fontlistQueries = [
@@ -11,11 +13,13 @@ const fontlistQueries = [
 	'version',
 	'type',
 ] as const;
-type FontlistQueries = typeof fontlistQueries[number] & keyof FontMetadata;
+type FontlistQueries = (typeof fontlistQueries)[number] & keyof FontMetadata;
 const isFontlistQuery = (query: string): query is FontlistQueries =>
 	fontlistQueries.includes(query as FontlistQueries);
 
 type Fontlist = Record<string, string | string[] | number[] | boolean>;
 
+type MetadataList = Record<string, IDResponse>;
+
 export { fontlistQueries, isFontlistQuery };
-export type { Fontlist, FontlistQueries };
+export type { Fontlist, FontlistQueries, MetadataList };
diff --git a/api/metadata/src/fontlist/update.ts b/api/metadata/src/fontlist/update.ts
@@ -1,26 +1,69 @@
-import type { FontsourceMetadata } from '../types';
-import { KV_TTL, METADATA_URL } from '../utils';
-import type { Fontlist, FontlistQueries } from './types';
+import { type FontVariants } from '../fonts/types';
+import type { FontMetadata, FontsourceMetadata } from '../types';
+import { METADATA_KEYS, METADATA_URL } from '../utils';
+import type { Fontlist, FontlistQueries, MetadataList } from './types';
 
-const updateMetadata = async (env: Env) => {
+const generateFontVariants = ({
+	id,
+	subsets,
+	weights,
+	styles,
+}: FontMetadata): FontVariants => {
+	const variants: FontVariants = {};
+
+	for (const weight of weights) {
+		variants[weight] = variants[weight] || {};
+
+		for (const style of styles) {
+			variants[weight][style] = variants[weight][style] || {};
+
+			for (const subset of subsets) {
+				variants[weight][style][subset] = {
+					url: {
+						woff2: `https://cdn.jsdelivr.net/fontsource/fonts/${id}@latest/${subset}-${weight}-${style}.woff2`,
+						woff: `https://cdn.jsdelivr.net/fontsource/fonts/${id}@latest/${subset}-${weight}-${style}.woff`,
+						ttf: `https://cdn.jsdelivr.net/fontsource/fonts/${id}@latest/${subset}-${weight}-${style}.ttf`,
+					},
+				};
+			}
+		}
+	}
+
+	return variants;
+};
+
+const updateMetadata = async (env: Env, ctx: ExecutionContext) => {
 	const response = await fetch(METADATA_URL);
-	const data = await response.json<FontsourceMetadata>();
+	const data = (await response.json()) as FontsourceMetadata;
+
+	const dataWithVariants: MetadataList = {};
+
+	for (const [key, value] of Object.entries(data)) {
+		const variants = generateFontVariants(value);
+		// Add variants to the metadata
+		dataWithVariants[key] = {
+			...value,
+			variable: Boolean(value.variable),
+			license: value.license.type,
+			variants,
+		};
+	}
 
 	// Save entire metadata into KV first
-	await env.FONTLIST.put('metadata', JSON.stringify(data), {
-		metadata: {
-			// We need to set a custom ttl for a stale-while-revalidate strategy
-			ttl: Date.now() / 1000 + KV_TTL,
-		},
-	});
-
-	return data;
+	ctx.waitUntil(
+		env.METADATA.put(METADATA_KEYS.fonts, JSON.stringify(dataWithVariants)),
+	);
+	return dataWithVariants;
 };
 
 // This updates the fontlist dataset for a given key
-const updateList = async (key: FontlistQueries, env: Env) => {
+const updateList = async (
+	key: FontlistQueries,
+	env: Env,
+	ctx: ExecutionContext,
+) => {
 	const response = await fetch(METADATA_URL);
-	const data = await response.json<FontsourceMetadata>();
+	const data = (await response.json()) as FontsourceMetadata;
 
 	// Depending on key, generate a fontlist object with respective values
 	const list: Fontlist = {};
@@ -31,11 +74,9 @@ const updateList = async (key: FontlistQueries, env: Env) => {
 	}
 
 	// Store the list in KV
-	await env.FONTLIST.put(key, JSON.stringify(list), {
-		metadata: {
-			ttl: Date.now() / 1000 + KV_TTL,
-		},
-	});
+	ctx.waitUntil(
+		env.METADATA.put(METADATA_KEYS.fontlist(key), JSON.stringify(list)),
+	);
 	return list;
 };
 

diff --git a/api/metadata/src/fonts/get.ts b/api/metadata/src/fonts/get.ts
@@ -1,43 +1,28 @@
-import { type TTLMetadata } from '../types';
-import type { ArrayMetadata, IDResponse } from './types';
-import { updateArrayMetadata, updateId } from './update';
+import { getMetadata } from '../fontlist/get';
+import { KV_TTL, METADATA_KEYS } from '../utils';
+import type { ArrayMetadata } from './types';
+import { updateArrayMetadata } from './update';
 
-const getOrUpdateArrayMetadata = async (env: Env, ctx: ExecutionContext) => {
-	const { value, metadata } = await env.FONTLIST.getWithMetadata<
-		ArrayMetadata,
-		TTLMetadata
-	>('metadata_arr', {
+const getArrayMetadata = async (env: Env, ctx: ExecutionContext) => {
+	const value = await env.METADATA.get<ArrayMetadata>(METADATA_KEYS.fonts_arr, {
 		type: 'json',
+		cacheTtl: KV_TTL,
 	});
 
 	if (!value) {
 		return await updateArrayMetadata(env, ctx);
 	}
 
-	// If the ttl is not set or the cache expiry is less than the current time, then return old value
-	// while revalidating the cache
-	if (!metadata?.ttl || metadata.ttl < Date.now() / 1000) {
-		ctx.waitUntil(updateArrayMetadata(env, ctx));
-	}
-
 	return value;
 };
 
-const getOrUpdateId = async (id: string, env: Env, ctx: ExecutionContext) => {
-	const { value, metadata } = await env.FONTS.getWithMetadata<
-		IDResponse,
-		TTLMetadata
-	>(id, { type: 'json' });
-
-	if (!value) {
-		return await updateId(id, env, ctx);
+const getId = async (id: string, env: Env, ctx: ExecutionContext) => {
+	const data = await getMetadata(env, ctx);
+	if (!data[id]) {
+		return;
 	}
 
-	if (!metadata?.ttl || metadata.ttl < Date.now() / 1000) {
-		ctx.waitUntil(updateId(id, env, ctx));
-	}
-
-	return value;
+	return data[id];
 };
 
-export { getOrUpdateArrayMetadata, getOrUpdateId };
+export { getArrayMetadata, getId };
diff --git a/api/metadata/src/fonts/router.ts b/api/metadata/src/fonts/router.ts
@@ -9,7 +9,7 @@ import {
 
 import { type CFRouterContext } from '../types';
 import { API_BROWSER_TTL, CF_EDGE_TTL } from '../utils';
-import { getOrUpdateArrayMetadata, getOrUpdateId } from './get';
+import { getArrayMetadata, getId } from './get';
 import { isFontsQueries } from './types';
 
 interface FontRequest extends IRequestStrict {
@@ -35,7 +35,7 @@ router.get('/v1/fonts', async (request, env, ctx) => {
 		'Cache-Control': `public, max-age=${API_BROWSER_TTL}`,
 		'CDN-Cache-Control': `max-age=${CF_EDGE_TTL}`,
 	};
-	const data = await getOrUpdateArrayMetadata(env, ctx);
+	const data = await getArrayMetadata(env, ctx);
 
 	// If no query string, return the entire list
 	if (url.searchParams.toString().length === 0) {
@@ -96,7 +96,7 @@ router.get('/v1/fonts/:id', withParams, async (request, env, ctx) => {
 		return response;
 	}
 
-	const data = await getOrUpdateId(id, env, ctx);
+	const data = await getId(id, env, ctx);
 	if (!data) {
 		return error(404, 'Not Found. Font does not exist.');
 	}