~james/core

core/web/src/lib/server/lookupEvents.ts -rw-r--r-- 4.1 KiB
452f17b7 — james chore(import): snapshot james.dad.town/core at f70c71c7696a 15 hours ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
import { Agent } from "undici";
import { didTag, handleTag, lookups, type LookupCache } from "#lib/api/lookupCache.ts";

// identity and account events always arrive. these two add a repo's own records
export const LOOKUP_COLLECTIONS = ["org.tangled.repo.manifest", "org.tangled.git.refs"];

// bobbin applies the same event from its own stream. a lookup refetched before it has
// would keep the old answer
const REEVICT_MS = 10_000;
const MIN_RETRY_MS = 1_000;
const MAX_RETRY_MS = 60_000;
const REPORT_MS = 60_000;

interface JetstreamEvent {
	did?: unknown;
	kind?: unknown;
	identity?: { handle?: unknown };
	commit?: { collection?: unknown };
}

export const tagsForEvent = (event: JetstreamEvent): string[] => {
	if (typeof event.did !== "string") return [];
	const tags = [didTag(event.did)];
	switch (event.kind) {
		case "identity":
			if (typeof event.identity?.handle === "string")
				tags.push(handleTag(event.identity.handle));
			return tags;
		case "account":
			return tags;
		case "commit":
			return LOOKUP_COLLECTIONS.includes(String(event.commit?.collection)) ? tags : [];
		default:
			return [];
	}
};

export const subscribeUrl = (url: string): string => {
	const parsed = new URL(url);
	for (const collection of LOOKUP_COLLECTIONS)
		parsed.searchParams.append("wantedCollections", collection);
	return parsed.href;
};

interface Socket {
	onopen: (() => void) | null;
	onmessage: ((message: { data: unknown }) => void) | null;
	onclose: (() => void) | null;
	close(): void;
}

export interface LookupEventsOptions {
	url: string;
	cache?: LookupCache;
	connect?: (url: string) => Socket;
}

export interface LookupEvents {
	readonly connected: boolean;
	close(): void;
}

// the cache is cleared on every open because events missed while down are gone
export const subscribeLookupEvents = ({
	url,
	cache = lookups,
	// node's websocket would use the global dispatcher, whose http/2 can't upgrade
	connect = (target) =>
		new WebSocket(target, { dispatcher: new Agent() } as never) as unknown as Socket
}: LookupEventsOptions): LookupEvents => {
	const target = subscribeUrl(url);
	let socket: Socket | undefined;
	let connected = false;
	let closed = false;
	let unreachable = false;
	let retryMs = MIN_RETRY_MS;

	const evict = (tags: string[]) => {
		cache.evict(tags);
		setTimeout(() => cache.evict(tags), REEVICT_MS).unref?.();
	};

	const open = () => {
		const current = connect(target);
		socket = current;
		current.onopen = () => {
			cache.clear();
			connected = true;
			unreachable = false;
			retryMs = MIN_RETRY_MS;
			console.info(`[lookups] following ${url}`);
		};
		current.onmessage = ({ data }) => {
			if (typeof data !== "string") return;
			let event: JetstreamEvent;
			try {
				event = JSON.parse(data) as JetstreamEvent;
			} catch {
				return;
			}
			const tags = tagsForEvent(event);
			if (tags.length > 0) evict(tags);
		};
		current.onclose = () => {
			if (socket !== current) return;
			if (connected) {
				console.warn(`[lookups] lost ${url}, cached lookups live out their ttl`);
			} else if (!unreachable) {
				console.warn(`[lookups] can't reach ${url}, retrying`);
				unreachable = true;
			}
			connected = false;
			if (closed) return;
			const wait = retryMs / 2 + Math.random() * (retryMs / 2);
			retryMs = Math.min(retryMs * 2, MAX_RETRY_MS);
			setTimeout(open, wait).unref?.();
		};
	};

	open();
	return {
		get connected() {
			return connected;
		},
		close() {
			closed = true;
			socket?.close();
		}
	};
};

export const startLookups = (config: { lookupCacheTtlMs: number; hydrantSubscribeUrl: string }) => {
	lookups.configure(config.lookupCacheTtlMs);
	if (config.lookupCacheTtlMs === 0) return;
	const events = subscribeLookupEvents({ url: config.hydrantSubscribeUrl });
	setInterval(() => {
		const { hits, misses, evictions, clears } = lookups.stats;
		if (hits + misses + evictions + clears === 0) return;
		console.info(
			`[lookups] hits=${hits} misses=${misses} evictions=${evictions} clears=${clears} entries=${lookups.size} connected=${events.connected}`
		);
		lookups.stats = { hits: 0, misses: 0, evictions: 0, clears: 0 };
	}, REPORT_MS).unref?.();
};