import type { ShardInfo, UrlEntry } from './types';
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8"?>';
const SITEMAP_SCHEMA = 'http://www.sitemaps.org/schemas/sitemap/0.9';
const TANGLED_EPOCH = new Date('2025-01-01T00:00:00Z');
export const PROTOCOL_MAX_URLS = 50000;
export const PROTOCOL_MAX_BYTES = 50 * 1024 * 1024; // 50 MB uncompressed
export function escapeXml(str: string): string {
return str
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/"/g, '"')
.replace(/'/g, ''');
}
export function sanitizeLastmod(rawDate?: string, referenceTime = new Date()): string {
if (!rawDate) {
return referenceTime.toISOString().split('T')[0];
}
const parsed = new Date(rawDate);
if (isNaN(parsed.getTime()) || parsed > referenceTime || parsed < TANGLED_EPOCH) {
return referenceTime.toISOString().split('T')[0];
}
return parsed.toISOString().split('T')[0];
}
export function measureXmlBytes(xml: string): number {
return new TextEncoder().encode(xml).byteLength;
}
export function buildUrlsetXml(urls: UrlEntry[]): string {
const body = urls
.map((u) => {
const parts = [` <loc>${escapeXml(u.loc)}</loc>`];
if (u.lastmod) parts.push(` <lastmod>${escapeXml(u.lastmod)}</lastmod>`);
if (u.changefreq) parts.push(` <changefreq>${u.changefreq}</changefreq>`);
if (u.priority !== undefined) parts.push(` <priority>${u.priority.toFixed(1)}</priority>`);
return ` <url>\n${parts.join('\n')}\n </url>`;
})
.join('\n');
return `${XML_HEADER}\n<urlset xmlns="${SITEMAP_SCHEMA}">\n${body}\n</urlset>\n`;
}
export function buildSitemapIndexXml(shards: ShardInfo[], baseUrl: string): string {
const body = shards
.map((shard) => {
const loc = `${baseUrl.replace(/\/+$/, '')}/sitemaps/${shard.filename}`;
return ` <sitemap>\n <loc>${escapeXml(loc)}</loc>\n <lastmod>${escapeXml(shard.lastmod)}</lastmod>\n </sitemap>`;
})
.join('\n');
return `${XML_HEADER}\n<sitemapindex xmlns="${SITEMAP_SCHEMA}">\n${body}\n</sitemapindex>\n`;
}
export async function gzipCompress(text: string): Promise<Uint8Array> {
const blob = new Blob([text]);
const stream = blob.stream().pipeThrough(new CompressionStream('gzip'));
const buffer = await new Response(stream).arrayBuffer();
return new Uint8Array(buffer);
}
export async function gzipDecompress(data: ArrayBuffer | Uint8Array): Promise<string> {
const blob = new Blob([data as any]);
const stream = blob.stream().pipeThrough(new DecompressionStream('gzip'));
return await new Response(stream).text();
}